From 20e6a84aca30a88387003f67c0f834ec61cebdfb Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 23 Nov 2015 17:16:54 +0000 Subject: [PATCH 01/65] Adding generated code for gRPC beta. --- gcloud_bigtable/_generated/annotations_pb2.py | 13 +- gcloud_bigtable/_generated/any_pb2.py | 17 +- .../_generated/bigtable_cluster_data_pb2.py | 21 +- .../bigtable_cluster_service_messages_pb2.py | 27 +-- .../bigtable_cluster_service_pb2.py | 192 ++++++++++++++-- .../_generated/bigtable_data_pb2.py | 71 +++--- .../bigtable_service_messages_pb2.py | 35 +-- .../_generated/bigtable_service_pb2.py | 148 ++++++++++-- .../_generated/bigtable_table_data_pb2.py | 25 +- .../bigtable_table_service_messages_pb2.py | 33 +-- .../_generated/bigtable_table_service_pb2.py | 213 ++++++++++++++++-- gcloud_bigtable/_generated/duration_pb2.py | 13 +- gcloud_bigtable/_generated/empty_pb2.py | 13 +- gcloud_bigtable/_generated/http_pb2.py | 29 +-- gcloud_bigtable/_generated/operations_pb2.py | 145 ++++++++++-- gcloud_bigtable/_generated/status_pb2.py | 15 +- gcloud_bigtable/_generated/timestamp_pb2.py | 13 +- 17 files changed, 779 insertions(+), 244 deletions(-) diff --git a/gcloud_bigtable/_generated/annotations_pb2.py b/gcloud_bigtable/_generated/annotations_pb2.py index 64545bc..8f98861 100644 --- a/gcloud_bigtable/_generated/annotations_pb2.py +++ b/gcloud_bigtable/_generated/annotations_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/api/annotations.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -21,7 +19,7 @@ name='google/api/annotations.proto', package='google.api', syntax='proto3', - serialized_pb=_b('\n\x1cgoogle/api/annotations.proto\x12\ngoogle.api\x1a\x15google/api/http.proto\x1a google/protobuf/descriptor.proto:E\n\x04http\x12\x1e.google.protobuf.MethodOptions\x18\xb0\xca\xbc\" \x01(\x0b\x32\x14.google.api.HttpRuleB$\n\x0e\x63om.google.apiB\x10\x41nnotationsProtoP\x01\x62\x06proto3') + serialized_pb=b'\n\x1cgoogle/api/annotations.proto\x12\ngoogle.api\x1a\x15google/api/http.proto\x1a google/protobuf/descriptor.proto:E\n\x04http\x12\x1e.google.protobuf.MethodOptions\x18\xb0\xca\xbc\" \x01(\x0b\x32\x14.google.api.HttpRuleB$\n\x0e\x63om.google.apiB\x10\x41nnotationsProtoP\x01\x62\x06proto3' , dependencies=[google_dot_api_dot_http__pb2.DESCRIPTOR,google_dot_protobuf_dot_descriptor__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -42,8 +40,11 @@ google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(http) DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.apiB\020AnnotationsProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\020AnnotationsProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/any_pb2.py b/gcloud_bigtable/_generated/any_pb2.py index 4c31a15..862ab1e 100644 --- a/gcloud_bigtable/_generated/any_pb2.py +++ b/gcloud_bigtable/_generated/any_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/protobuf/any.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/protobuf/any.proto', package='google.protobuf', syntax='proto3', - serialized_pb=_b('\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"&\n\x03\x41ny\x12\x10\n\x08type_url\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\x42$\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01\xa0\x01\x01\x62\x06proto3') + serialized_pb=b'\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"&\n\x03\x41ny\x12\x10\n\x08type_url\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\x42$\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01\xa0\x01\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -36,14 +34,14 @@ _descriptor.FieldDescriptor( name='type_url', full_name='google.protobuf.Any.type_url', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='value', full_name='google.protobuf.Any.value', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -74,8 +72,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\010AnyProtoP\001\240\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\010AnyProtoP\001\240\001\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py index 544af89..6be8475 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message @@ -23,7 +21,7 @@ name='google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto', package='google.bigtable.admin.cluster.v1', syntax='proto3', - serialized_pb=_b('\n""" __metaclass__ = abc.ABCMeta @@ -118,43 +119,43 @@ def early_adopter_create_BigtableClusterService_server(servicer, port, private_k import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 import gcloud_bigtable._generated.operations_pb2 method_service_descriptions = { - "CreateCluster": utilities.unary_unary_service_description( + "CreateCluster": alpha_utilities.unary_unary_service_description( servicer.CreateCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, ), - "DeleteCluster": utilities.unary_unary_service_description( + "DeleteCluster": alpha_utilities.unary_unary_service_description( servicer.DeleteCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "GetCluster": utilities.unary_unary_service_description( + "GetCluster": alpha_utilities.unary_unary_service_description( servicer.GetCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, ), - "ListClusters": utilities.unary_unary_service_description( + "ListClusters": alpha_utilities.unary_unary_service_description( servicer.ListClusters, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, ), - "ListZones": utilities.unary_unary_service_description( + "ListZones": alpha_utilities.unary_unary_service_description( servicer.ListZones, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, ), - "UndeleteCluster": utilities.unary_unary_service_description( + "UndeleteCluster": alpha_utilities.unary_unary_service_description( servicer.UndeleteCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, ), - "UpdateCluster": utilities.unary_unary_service_description( + "UpdateCluster": alpha_utilities.unary_unary_service_description( servicer.UpdateCluster, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, ), } - return implementations.server("google.bigtable.admin.cluster.v1.BigtableClusterService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) + return early_adopter_implementations.server("google.bigtable.admin.cluster.v1.BigtableClusterService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 @@ -171,34 +172,181 @@ def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transf import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 import gcloud_bigtable._generated.operations_pb2 method_invocation_descriptions = { - "CreateCluster": utilities.unary_unary_invocation_description( + "CreateCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, ), - "DeleteCluster": utilities.unary_unary_invocation_description( + "DeleteCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "GetCluster": utilities.unary_unary_invocation_description( + "GetCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, ), - "ListClusters": utilities.unary_unary_invocation_description( + "ListClusters": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, ), - "ListZones": utilities.unary_unary_invocation_description( + "ListZones": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, ), - "UndeleteCluster": utilities.unary_unary_invocation_description( + "UndeleteCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, gcloud_bigtable._generated.operations_pb2.Operation.FromString, ), - "UpdateCluster": utilities.unary_unary_invocation_description( + "UpdateCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, ), } - return implementations.stub("google.bigtable.admin.cluster.v1.BigtableClusterService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + return early_adopter_implementations.stub("google.bigtable.admin.cluster.v1.BigtableClusterService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + +class BetaBigtableClusterServiceServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def ListZones(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def GetCluster(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ListClusters(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CreateCluster(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def UpdateCluster(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteCluster(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def UndeleteCluster(self, request, context): + raise NotImplementedError() + +class BetaBigtableClusterServiceStub(object): + """The interface to which stubs will conform.""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def ListZones(self, request, timeout): + raise NotImplementedError() + ListZones.future = None + @abc.abstractmethod + def GetCluster(self, request, timeout): + raise NotImplementedError() + GetCluster.future = None + @abc.abstractmethod + def ListClusters(self, request, timeout): + raise NotImplementedError() + ListClusters.future = None + @abc.abstractmethod + def CreateCluster(self, request, timeout): + raise NotImplementedError() + CreateCluster.future = None + @abc.abstractmethod + def UpdateCluster(self, request, timeout): + raise NotImplementedError() + UpdateCluster.future = None + @abc.abstractmethod + def DeleteCluster(self, request, timeout): + raise NotImplementedError() + DeleteCluster.future = None + @abc.abstractmethod + def UndeleteCluster(self, request, timeout): + raise NotImplementedError() + UndeleteCluster.future = None + +def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.operations_pb2 + request_deserializers = { + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + } + response_serializers = { + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + } + method_implementations = { + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): face_utilities.unary_unary_inline(servicer.CreateCluster), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): face_utilities.unary_unary_inline(servicer.DeleteCluster), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): face_utilities.unary_unary_inline(servicer.GetCluster), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): face_utilities.unary_unary_inline(servicer.ListClusters), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): face_utilities.unary_unary_inline(servicer.ListZones), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): face_utilities.unary_unary_inline(servicer.UndeleteCluster), + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): face_utilities.unary_unary_inline(servicer.UpdateCluster), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + +def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_data_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 + import gcloud_bigtable._generated.operations_pb2 + request_serializers = { + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + } + response_deserializers = { + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + } + cardinalities = { + 'CreateCluster': cardinality.Cardinality.UNARY_UNARY, + 'DeleteCluster': cardinality.Cardinality.UNARY_UNARY, + 'GetCluster': cardinality.Cardinality.UNARY_UNARY, + 'ListClusters': cardinality.Cardinality.UNARY_UNARY, + 'ListZones': cardinality.Cardinality.UNARY_UNARY, + 'UndeleteCluster': cardinality.Cardinality.UNARY_UNARY, + 'UpdateCluster': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'google.bigtable.admin.cluster.v1.BigtableClusterService', cardinalities, options=stub_options) # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_data_pb2.py b/gcloud_bigtable/_generated/bigtable_data_pb2.py index cb8f0b1..58a3b7b 100644 --- a/gcloud_bigtable/_generated/bigtable_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_data_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/v1/bigtable_data.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/bigtable/v1/bigtable_data.proto', package='google.bigtable.v1', syntax='proto3', - serialized_pb=_b('\n&google/bigtable/v1/bigtable_data.proto\x12\x12google.bigtable.v1\"@\n\x03Row\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12,\n\x08\x66\x61milies\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Family\"C\n\x06\x46\x61mily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Column\"D\n\x06\x43olumn\x12\x11\n\tqualifier\x18\x01 \x01(\x0c\x12\'\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32\x18.google.bigtable.v1.Cell\"?\n\x04\x43\x65ll\x12\x18\n\x10timestamp_micros\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06labels\x18\x03 \x03(\t\".\n\x08RowRange\x12\x11\n\tstart_key\x18\x02 \x01(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\"\xd6\x01\n\x0b\x43olumnRange\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12#\n\x19start_qualifier_inclusive\x18\x02 \x01(\x0cH\x00\x12#\n\x19start_qualifier_exclusive\x18\x03 \x01(\x0cH\x00\x12!\n\x17\x65nd_qualifier_inclusive\x18\x04 \x01(\x0cH\x01\x12!\n\x17\x65nd_qualifier_exclusive\x18\x05 \x01(\x0cH\x01\x42\x11\n\x0fstart_qualifierB\x0f\n\rend_qualifier\"N\n\x0eTimestampRange\x12\x1e\n\x16start_timestamp_micros\x18\x01 \x01(\x03\x12\x1c\n\x14\x65nd_timestamp_micros\x18\x02 \x01(\x03\"\xa8\x01\n\nValueRange\x12\x1f\n\x15start_value_inclusive\x18\x01 \x01(\x0cH\x00\x12\x1f\n\x15start_value_exclusive\x18\x02 \x01(\x0cH\x00\x12\x1d\n\x13\x65nd_value_inclusive\x18\x03 \x01(\x0cH\x01\x12\x1d\n\x13\x65nd_value_exclusive\x18\x04 \x01(\x0cH\x01\x42\r\n\x0bstart_valueB\x0b\n\tend_value\"\xdf\x08\n\tRowFilter\x12\x34\n\x05\x63hain\x18\x01 \x01(\x0b\x32#.google.bigtable.v1.RowFilter.ChainH\x00\x12>\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3') + serialized_pb=b'\n&google/bigtable/v1/bigtable_data.proto\x12\x12google.bigtable.v1\"@\n\x03Row\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12,\n\x08\x66\x61milies\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Family\"C\n\x06\x46\x61mily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Column\"D\n\x06\x43olumn\x12\x11\n\tqualifier\x18\x01 \x01(\x0c\x12\'\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32\x18.google.bigtable.v1.Cell\"?\n\x04\x43\x65ll\x12\x18\n\x10timestamp_micros\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06labels\x18\x03 \x03(\t\".\n\x08RowRange\x12\x11\n\tstart_key\x18\x02 \x01(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\"\xd6\x01\n\x0b\x43olumnRange\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12#\n\x19start_qualifier_inclusive\x18\x02 \x01(\x0cH\x00\x12#\n\x19start_qualifier_exclusive\x18\x03 \x01(\x0cH\x00\x12!\n\x17\x65nd_qualifier_inclusive\x18\x04 \x01(\x0cH\x01\x12!\n\x17\x65nd_qualifier_exclusive\x18\x05 \x01(\x0cH\x01\x42\x11\n\x0fstart_qualifierB\x0f\n\rend_qualifier\"N\n\x0eTimestampRange\x12\x1e\n\x16start_timestamp_micros\x18\x01 \x01(\x03\x12\x1c\n\x14\x65nd_timestamp_micros\x18\x02 \x01(\x03\"\xa8\x01\n\nValueRange\x12\x1f\n\x15start_value_inclusive\x18\x01 \x01(\x0cH\x00\x12\x1f\n\x15start_value_exclusive\x18\x02 \x01(\x0cH\x00\x12\x1d\n\x13\x65nd_value_inclusive\x18\x03 \x01(\x0cH\x01\x12\x1d\n\x13\x65nd_value_exclusive\x18\x04 \x01(\x0cH\x01\x42\r\n\x0bstart_valueB\x0b\n\tend_value\"\xdf\x08\n\tRowFilter\x12\x34\n\x05\x63hain\x18\x01 \x01(\x0b\x32#.google.bigtable.v1.RowFilter.ChainH\x00\x12>\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -36,7 +34,7 @@ _descriptor.FieldDescriptor( name='key', full_name='google.bigtable.v1.Row.key', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -74,7 +72,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.v1.Family.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -112,7 +110,7 @@ _descriptor.FieldDescriptor( name='qualifier', full_name='google.bigtable.v1.Column.qualifier', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -157,7 +155,7 @@ _descriptor.FieldDescriptor( name='value', full_name='google.bigtable.v1.Cell.value', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -195,14 +193,14 @@ _descriptor.FieldDescriptor( name='start_key', full_name='google.bigtable.v1.RowRange.start_key', index=0, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='end_key', full_name='google.bigtable.v1.RowRange.end_key', index=1, number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -233,35 +231,35 @@ _descriptor.FieldDescriptor( name='family_name', full_name='google.bigtable.v1.ColumnRange.family_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='start_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_inclusive', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='start_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_exclusive', index=2, number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='end_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_inclusive', index=3, number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='end_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_exclusive', index=4, number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -336,28 +334,28 @@ _descriptor.FieldDescriptor( name='start_value_inclusive', full_name='google.bigtable.v1.ValueRange.start_value_inclusive', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='start_value_exclusive', full_name='google.bigtable.v1.ValueRange.start_value_exclusive', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='end_value_inclusive', full_name='google.bigtable.v1.ValueRange.end_value_inclusive', index=2, number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='end_value_exclusive', full_name='google.bigtable.v1.ValueRange.end_value_exclusive', index=3, number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -540,7 +538,7 @@ _descriptor.FieldDescriptor( name='row_key_regex_filter', full_name='google.bigtable.v1.RowFilter.row_key_regex_filter', index=6, number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -554,14 +552,14 @@ _descriptor.FieldDescriptor( name='family_name_regex_filter', full_name='google.bigtable.v1.RowFilter.family_name_regex_filter', index=8, number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='column_qualifier_regex_filter', full_name='google.bigtable.v1.RowFilter.column_qualifier_regex_filter', index=9, number=6, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -582,7 +580,7 @@ _descriptor.FieldDescriptor( name='value_regex_filter', full_name='google.bigtable.v1.RowFilter.value_regex_filter', index=12, number=9, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -624,7 +622,7 @@ _descriptor.FieldDescriptor( name='apply_label_transformer', full_name='google.bigtable.v1.RowFilter.apply_label_transformer', index=18, number=19, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -658,14 +656,14 @@ _descriptor.FieldDescriptor( name='family_name', full_name='google.bigtable.v1.Mutation.SetCell.family_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='column_qualifier', full_name='google.bigtable.v1.Mutation.SetCell.column_qualifier', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -679,7 +677,7 @@ _descriptor.FieldDescriptor( name='value', full_name='google.bigtable.v1.Mutation.SetCell.value', index=3, number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -709,14 +707,14 @@ _descriptor.FieldDescriptor( name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.family_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='column_qualifier', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.column_qualifier', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -753,7 +751,7 @@ _descriptor.FieldDescriptor( name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromFamily.family_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -861,21 +859,21 @@ _descriptor.FieldDescriptor( name='family_name', full_name='google.bigtable.v1.ReadModifyWriteRule.family_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='column_qualifier', full_name='google.bigtable.v1.ReadModifyWriteRule.column_qualifier', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='append_value', full_name='google.bigtable.v1.ReadModifyWriteRule.append_value', index=2, number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -1177,8 +1175,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\021BigtableDataProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\021BigtableDataProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py index c07deac..9b5b5d4 100644 --- a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/v1/bigtable_service_messages.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -20,7 +18,7 @@ name='google/bigtable/v1/bigtable_service_messages.proto', package='google.bigtable.v1', syntax='proto3', - serialized_pb=_b('\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\"\xdc\x01\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3') + serialized_pb=b'\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\"\xdc\x01\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3' , dependencies=[google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -38,14 +36,14 @@ _descriptor.FieldDescriptor( name='table_name', full_name='google.bigtable.v1.ReadRowsRequest.table_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.ReadRowsRequest.row_key', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -154,7 +152,7 @@ _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.ReadRowsResponse.row_key', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -192,7 +190,7 @@ _descriptor.FieldDescriptor( name='table_name', full_name='google.bigtable.v1.SampleRowKeysRequest.table_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -223,7 +221,7 @@ _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.SampleRowKeysResponse.row_key', index=0, number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -261,14 +259,14 @@ _descriptor.FieldDescriptor( name='table_name', full_name='google.bigtable.v1.MutateRowRequest.table_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.MutateRowRequest.row_key', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -306,14 +304,14 @@ _descriptor.FieldDescriptor( name='table_name', full_name='google.bigtable.v1.CheckAndMutateRowRequest.table_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.CheckAndMutateRowRequest.row_key', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -396,14 +394,14 @@ _descriptor.FieldDescriptor( name='table_name', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.table_name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='row_key', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.row_key', index=1, number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -530,8 +528,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\034BigtableServiceMessagesProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\034BigtableServiceMessagesProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index 4e7e7e4..bd0900f 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/v1/bigtable_service.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -23,7 +21,7 @@ name='google/bigtable/v1/bigtable_service.proto', package='google.bigtable.v1', syntax='proto3', - serialized_pb=_b('\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xb0\x07\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3') + serialized_pb=b'\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xb0\x07\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3' , dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -33,10 +31,13 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\025BigtableServicesProtoP\001\210\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\025BigtableServicesProtoP\001\210\001\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities class EarlyAdopterBigtableServiceServicer(object): """""" __metaclass__ = abc.ABCMeta @@ -99,33 +100,33 @@ def early_adopter_create_BigtableService_server(servicer, port, private_key=None import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 method_service_descriptions = { - "CheckAndMutateRow": utilities.unary_unary_service_description( + "CheckAndMutateRow": alpha_utilities.unary_unary_service_description( servicer.CheckAndMutateRow, gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, ), - "MutateRow": utilities.unary_unary_service_description( + "MutateRow": alpha_utilities.unary_unary_service_description( servicer.MutateRow, gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "ReadModifyWriteRow": utilities.unary_unary_service_description( + "ReadModifyWriteRow": alpha_utilities.unary_unary_service_description( servicer.ReadModifyWriteRow, gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, ), - "ReadRows": utilities.unary_stream_service_description( + "ReadRows": alpha_utilities.unary_stream_service_description( servicer.ReadRows, gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, ), - "SampleRowKeys": utilities.unary_stream_service_description( + "SampleRowKeys": alpha_utilities.unary_stream_service_description( servicer.SampleRowKeys, gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, ), } - return implementations.server("google.bigtable.v1.BigtableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) + return early_adopter_implementations.server("google.bigtable.v1.BigtableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 @@ -138,26 +139,137 @@ def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=N import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 method_invocation_descriptions = { - "CheckAndMutateRow": utilities.unary_unary_invocation_description( + "CheckAndMutateRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, ), - "MutateRow": utilities.unary_unary_invocation_description( + "MutateRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "ReadModifyWriteRow": utilities.unary_unary_invocation_description( + "ReadModifyWriteRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, ), - "ReadRows": utilities.unary_stream_invocation_description( + "ReadRows": alpha_utilities.unary_stream_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, ), - "SampleRowKeys": utilities.unary_stream_invocation_description( + "SampleRowKeys": alpha_utilities.unary_stream_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, ), } - return implementations.stub("google.bigtable.v1.BigtableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + return early_adopter_implementations.stub("google.bigtable.v1.BigtableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + +class BetaBigtableServiceServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def ReadRows(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def SampleRowKeys(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def MutateRow(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CheckAndMutateRow(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ReadModifyWriteRow(self, request, context): + raise NotImplementedError() + +class BetaBigtableServiceStub(object): + """The interface to which stubs will conform.""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def ReadRows(self, request, timeout): + raise NotImplementedError() + @abc.abstractmethod + def SampleRowKeys(self, request, timeout): + raise NotImplementedError() + @abc.abstractmethod + def MutateRow(self, request, timeout): + raise NotImplementedError() + MutateRow.future = None + @abc.abstractmethod + def CheckAndMutateRow(self, request, timeout): + raise NotImplementedError() + CheckAndMutateRow.future = None + @abc.abstractmethod + def ReadModifyWriteRow(self, request, timeout): + raise NotImplementedError() + ReadModifyWriteRow.future = None + +def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_data_pb2 + request_deserializers = { + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.MutateRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsRequest.FromString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, + } + response_serializers = { + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, + } + method_implementations = { + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): face_utilities.unary_unary_inline(servicer.CheckAndMutateRow), + ('google.bigtable.v1.BigtableService', 'MutateRow'): face_utilities.unary_unary_inline(servicer.MutateRow), + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): face_utilities.unary_unary_inline(servicer.ReadModifyWriteRow), + ('google.bigtable.v1.BigtableService', 'ReadRows'): face_utilities.unary_stream_inline(servicer.ReadRows), + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): face_utilities.unary_stream_inline(servicer.SampleRowKeys), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + +def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_data_pb2 + request_serializers = { + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, + } + response_deserializers = { + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.FromString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.FromString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, + } + cardinalities = { + 'CheckAndMutateRow': cardinality.Cardinality.UNARY_UNARY, + 'MutateRow': cardinality.Cardinality.UNARY_UNARY, + 'ReadModifyWriteRow': cardinality.Cardinality.UNARY_UNARY, + 'ReadRows': cardinality.Cardinality.UNARY_STREAM, + 'SampleRowKeys': cardinality.Cardinality.UNARY_STREAM, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'google.bigtable.v1.BigtableService', cardinalities, options=stub_options) # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py index 22cc067..5cbc242 100644 --- a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/admin/table/v1/bigtable_table_data.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -21,7 +19,7 @@ name='google/bigtable/admin/table/v1/bigtable_table_data.proto', package='google.bigtable.admin.table.v1', syntax='proto3', - serialized_pb=_b('\n8google/bigtable/admin/table/v1/bigtable_table_data.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a#google/longrunning/operations.proto\x1a\x1egoogle/protobuf/duration.proto\"\xfd\x02\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x11\x63urrent_operation\x18\x02 \x01(\x0b\x32\x1d.google.longrunning.Operation\x12R\n\x0f\x63olumn_families\x18\x03 \x03(\x0b\x32\x39.google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry\x12O\n\x0bgranularity\x18\x04 \x01(\x0e\x32:.google.bigtable.admin.table.v1.Table.TimestampGranularity\x1a\x63\n\x13\x43olumnFamiliesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily:\x02\x38\x01\"\"\n\x14TimestampGranularity\x12\n\n\x06MILLIS\x10\x00\"l\n\x0c\x43olumnFamily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rgc_expression\x18\x02 \x01(\t\x12\x37\n\x07gc_rule\x18\x03 \x01(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\"\xed\x02\n\x06GcRule\x12\x1a\n\x10max_num_versions\x18\x01 \x01(\x05H\x00\x12,\n\x07max_age\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12K\n\x0cintersection\x18\x03 \x01(\x0b\x32\x33.google.bigtable.admin.table.v1.GcRule.IntersectionH\x00\x12=\n\x05union\x18\x04 \x01(\x0b\x32,.google.bigtable.admin.table.v1.GcRule.UnionH\x00\x1a\x45\n\x0cIntersection\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\x1a>\n\x05Union\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRuleB\x06\n\x04ruleB>\n\"com.google.bigtable.admin.table.v1B\x16\x42igtableTableDataProtoP\x01\x62\x06proto3') + serialized_pb=b'\n8google/bigtable/admin/table/v1/bigtable_table_data.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a#google/longrunning/operations.proto\x1a\x1egoogle/protobuf/duration.proto\"\xfd\x02\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x11\x63urrent_operation\x18\x02 \x01(\x0b\x32\x1d.google.longrunning.Operation\x12R\n\x0f\x63olumn_families\x18\x03 \x03(\x0b\x32\x39.google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry\x12O\n\x0bgranularity\x18\x04 \x01(\x0e\x32:.google.bigtable.admin.table.v1.Table.TimestampGranularity\x1a\x63\n\x13\x43olumnFamiliesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily:\x02\x38\x01\"\"\n\x14TimestampGranularity\x12\n\n\x06MILLIS\x10\x00\"l\n\x0c\x43olumnFamily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rgc_expression\x18\x02 \x01(\t\x12\x37\n\x07gc_rule\x18\x03 \x01(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\"\xed\x02\n\x06GcRule\x12\x1a\n\x10max_num_versions\x18\x01 \x01(\x05H\x00\x12,\n\x07max_age\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12K\n\x0cintersection\x18\x03 \x01(\x0b\x32\x33.google.bigtable.admin.table.v1.GcRule.IntersectionH\x00\x12=\n\x05union\x18\x04 \x01(\x0b\x32,.google.bigtable.admin.table.v1.GcRule.UnionH\x00\x1a\x45\n\x0cIntersection\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\x1a>\n\x05Union\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRuleB\x06\n\x04ruleB>\n\"com.google.bigtable.admin.table.v1B\x16\x42igtableTableDataProtoP\x01\x62\x06proto3' , dependencies=[google_dot_longrunning_dot_operations__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -57,7 +55,7 @@ _descriptor.FieldDescriptor( name='key', full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -74,7 +72,7 @@ nested_types=[], enum_types=[ ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), b'8\001'), is_extendable=False, syntax='proto3', extension_ranges=[], @@ -94,7 +92,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.Table.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -147,14 +145,14 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.ColumnFamily.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='gc_expression', full_name='google.bigtable.admin.table.v1.ColumnFamily.gc_expression', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -373,10 +371,13 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B\026BigtableTableDataProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\026BigtableTableDataProtoP\001') _TABLE_COLUMNFAMILIESENTRY.has_options = True -_TABLE_COLUMNFAMILIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_TABLE_COLUMNFAMILIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), b'8\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py index 30d8703..245a0f0 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/admin/table/v1/bigtable_table_service_messages.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -20,7 +18,7 @@ name='google/bigtable/admin/table/v1/bigtable_table_service_messages.proto', package='google.bigtable.admin.table.v1', syntax='proto3', - serialized_pb=_b('\nDgoogle/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\"\x86\x01\n\x12\x43reateTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08table_id\x18\x02 \x01(\t\x12\x34\n\x05table\x18\x03 \x01(\x0b\x32%.google.bigtable.admin.table.v1.Table\x12\x1a\n\x12initial_split_keys\x18\x04 \x03(\t\"!\n\x11ListTablesRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"K\n\x12ListTablesResponse\x12\x35\n\x06tables\x18\x01 \x03(\x0b\x32%.google.bigtable.admin.table.v1.Table\"\x1f\n\x0fGetTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\"\n\x12\x44\x65leteTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x12RenameTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\"\x88\x01\n\x19\x43reateColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_family_id\x18\x02 \x01(\t\x12\x43\n\rcolumn_family\x18\x03 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily\")\n\x19\x44\x65leteColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\tBI\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\x01\x62\x06proto3') + serialized_pb=b'\nDgoogle/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\"\x86\x01\n\x12\x43reateTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08table_id\x18\x02 \x01(\t\x12\x34\n\x05table\x18\x03 \x01(\x0b\x32%.google.bigtable.admin.table.v1.Table\x12\x1a\n\x12initial_split_keys\x18\x04 \x03(\t\"!\n\x11ListTablesRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"K\n\x12ListTablesResponse\x12\x35\n\x06tables\x18\x01 \x03(\x0b\x32%.google.bigtable.admin.table.v1.Table\"\x1f\n\x0fGetTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\"\n\x12\x44\x65leteTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x12RenameTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\"\x88\x01\n\x19\x43reateColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_family_id\x18\x02 \x01(\t\x12\x43\n\rcolumn_family\x18\x03 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily\")\n\x19\x44\x65leteColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\tBI\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\x01\x62\x06proto3' , dependencies=[google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -38,14 +36,14 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.CreateTableRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='table_id', full_name='google.bigtable.admin.table.v1.CreateTableRequest.table_id', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -90,7 +88,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.ListTablesRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -152,7 +150,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.GetTableRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -183,7 +181,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.DeleteTableRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -214,14 +212,14 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.RenameTableRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='new_id', full_name='google.bigtable.admin.table.v1.RenameTableRequest.new_id', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -252,14 +250,14 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='column_family_id', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.column_family_id', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -297,7 +295,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.bigtable.admin.table.v1.DeleteColumnFamilyRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -387,8 +385,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py index 5dbf54e..4567385 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/bigtable/admin/table/v1/bigtable_table_service.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -23,7 +21,7 @@ name='google/bigtable/admin/table/v1/bigtable_table_service.proto', package='google.bigtable.admin.table.v1', syntax='proto3', - serialized_pb=_b('\n;google/bigtable/admin/table/v1/bigtable_table_service.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x1cgoogle/api/annotations.proto\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\x1a\x44google/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\x89\x0b\n\x14\x42igtableTableService\x12\xa4\x01\n\x0b\x43reateTable\x12\x32.google.bigtable.admin.table.v1.CreateTableRequest\x1a%.google.bigtable.admin.table.v1.Table\":\x82\xd3\xe4\x93\x02\x34\"//v1/{name=projects/*/zones/*/clusters/*}/tables:\x01*\x12\xac\x01\n\nListTables\x12\x31.google.bigtable.admin.table.v1.ListTablesRequest\x1a\x32.google.bigtable.admin.table.v1.ListTablesResponse\"7\x82\xd3\xe4\x93\x02\x31\x12//v1/{name=projects/*/zones/*/clusters/*}/tables\x12\x9d\x01\n\x08GetTable\x12/.google.bigtable.admin.table.v1.GetTableRequest\x1a%.google.bigtable.admin.table.v1.Table\"9\x82\xd3\xe4\x93\x02\x33\x12\x31/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x94\x01\n\x0b\x44\x65leteTable\x12\x32.google.bigtable.admin.table.v1.DeleteTableRequest\x1a\x16.google.protobuf.Empty\"9\x82\xd3\xe4\x93\x02\x33*1/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x9e\x01\n\x0bRenameTable\x12\x32.google.bigtable.admin.table.v1.RenameTableRequest\x1a\x16.google.protobuf.Empty\"C\x82\xd3\xe4\x93\x02=\"8/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename:\x01*\x12\xca\x01\n\x12\x43reateColumnFamily\x12\x39.google.bigtable.admin.table.v1.CreateColumnFamilyRequest\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"K\x82\xd3\xe4\x93\x02\x45\"@/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies:\x01*\x12\xbf\x01\n\x12UpdateColumnFamily\x12,.google.bigtable.admin.table.v1.ColumnFamily\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"M\x82\xd3\xe4\x93\x02G\x1a\x42/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}:\x01*\x12\xb3\x01\n\x12\x44\x65leteColumnFamily\x12\x39.google.bigtable.admin.table.v1.DeleteColumnFamilyRequest\x1a\x16.google.protobuf.Empty\"J\x82\xd3\xe4\x93\x02\x44*B/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}BB\n\"com.google.bigtable.admin.table.v1B\x1a\x42igtableTableServicesProtoP\x01\x62\x06proto3') + serialized_pb=b'\n;google/bigtable/admin/table/v1/bigtable_table_service.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x1cgoogle/api/annotations.proto\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\x1a\x44google/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\x89\x0b\n\x14\x42igtableTableService\x12\xa4\x01\n\x0b\x43reateTable\x12\x32.google.bigtable.admin.table.v1.CreateTableRequest\x1a%.google.bigtable.admin.table.v1.Table\":\x82\xd3\xe4\x93\x02\x34\"//v1/{name=projects/*/zones/*/clusters/*}/tables:\x01*\x12\xac\x01\n\nListTables\x12\x31.google.bigtable.admin.table.v1.ListTablesRequest\x1a\x32.google.bigtable.admin.table.v1.ListTablesResponse\"7\x82\xd3\xe4\x93\x02\x31\x12//v1/{name=projects/*/zones/*/clusters/*}/tables\x12\x9d\x01\n\x08GetTable\x12/.google.bigtable.admin.table.v1.GetTableRequest\x1a%.google.bigtable.admin.table.v1.Table\"9\x82\xd3\xe4\x93\x02\x33\x12\x31/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x94\x01\n\x0b\x44\x65leteTable\x12\x32.google.bigtable.admin.table.v1.DeleteTableRequest\x1a\x16.google.protobuf.Empty\"9\x82\xd3\xe4\x93\x02\x33*1/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x9e\x01\n\x0bRenameTable\x12\x32.google.bigtable.admin.table.v1.RenameTableRequest\x1a\x16.google.protobuf.Empty\"C\x82\xd3\xe4\x93\x02=\"8/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename:\x01*\x12\xca\x01\n\x12\x43reateColumnFamily\x12\x39.google.bigtable.admin.table.v1.CreateColumnFamilyRequest\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"K\x82\xd3\xe4\x93\x02\x45\"@/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies:\x01*\x12\xbf\x01\n\x12UpdateColumnFamily\x12,.google.bigtable.admin.table.v1.ColumnFamily\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"M\x82\xd3\xe4\x93\x02G\x1a\x42/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}:\x01*\x12\xb3\x01\n\x12\x44\x65leteColumnFamily\x12\x39.google.bigtable.admin.table.v1.DeleteColumnFamilyRequest\x1a\x16.google.protobuf.Empty\"J\x82\xd3\xe4\x93\x02\x44*B/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}BB\n\"com.google.bigtable.admin.table.v1B\x1a\x42igtableTableServicesProtoP\x01\x62\x06proto3' , dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -33,10 +31,13 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B\032BigtableTableServicesProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\032BigtableTableServicesProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities class EarlyAdopterBigtableTableServiceServicer(object): """""" __metaclass__ = abc.ABCMeta @@ -126,48 +127,48 @@ def early_adopter_create_BigtableTableService_server(servicer, port, private_key import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.empty_pb2 method_service_descriptions = { - "CreateColumnFamily": utilities.unary_unary_service_description( + "CreateColumnFamily": alpha_utilities.unary_unary_service_description( servicer.CreateColumnFamily, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, ), - "CreateTable": utilities.unary_unary_service_description( + "CreateTable": alpha_utilities.unary_unary_service_description( servicer.CreateTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, ), - "DeleteColumnFamily": utilities.unary_unary_service_description( + "DeleteColumnFamily": alpha_utilities.unary_unary_service_description( servicer.DeleteColumnFamily, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "DeleteTable": utilities.unary_unary_service_description( + "DeleteTable": alpha_utilities.unary_unary_service_description( servicer.DeleteTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "GetTable": utilities.unary_unary_service_description( + "GetTable": alpha_utilities.unary_unary_service_description( servicer.GetTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.FromString, gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, ), - "ListTables": utilities.unary_unary_service_description( + "ListTables": alpha_utilities.unary_unary_service_description( servicer.ListTables, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, ), - "RenameTable": utilities.unary_unary_service_description( + "RenameTable": alpha_utilities.unary_unary_service_description( servicer.RenameTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "UpdateColumnFamily": utilities.unary_unary_service_description( + "UpdateColumnFamily": alpha_utilities.unary_unary_service_description( servicer.UpdateColumnFamily, gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, ), } - return implementations.server("google.bigtable.admin.table.v1.BigtableTableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) + return early_adopter_implementations.server("google.bigtable.admin.table.v1.BigtableTableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) def early_adopter_create_BigtableTableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 @@ -186,38 +187,202 @@ def early_adopter_create_BigtableTableService_stub(host, port, metadata_transfor import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.empty_pb2 method_invocation_descriptions = { - "CreateColumnFamily": utilities.unary_unary_invocation_description( + "CreateColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, ), - "CreateTable": utilities.unary_unary_invocation_description( + "CreateTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, ), - "DeleteColumnFamily": utilities.unary_unary_invocation_description( + "DeleteColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "DeleteTable": utilities.unary_unary_invocation_description( + "DeleteTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "GetTable": utilities.unary_unary_invocation_description( + "GetTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, ), - "ListTables": utilities.unary_unary_invocation_description( + "ListTables": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, ), - "RenameTable": utilities.unary_unary_invocation_description( + "RenameTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "UpdateColumnFamily": utilities.unary_unary_invocation_description( + "UpdateColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, ), } - return implementations.stub("google.bigtable.admin.table.v1.BigtableTableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + return early_adopter_implementations.stub("google.bigtable.admin.table.v1.BigtableTableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + +class BetaBigtableTableServiceServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def CreateTable(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ListTables(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def GetTable(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteTable(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def RenameTable(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CreateColumnFamily(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def UpdateColumnFamily(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteColumnFamily(self, request, context): + raise NotImplementedError() + +class BetaBigtableTableServiceStub(object): + """The interface to which stubs will conform.""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def CreateTable(self, request, timeout): + raise NotImplementedError() + CreateTable.future = None + @abc.abstractmethod + def ListTables(self, request, timeout): + raise NotImplementedError() + ListTables.future = None + @abc.abstractmethod + def GetTable(self, request, timeout): + raise NotImplementedError() + GetTable.future = None + @abc.abstractmethod + def DeleteTable(self, request, timeout): + raise NotImplementedError() + DeleteTable.future = None + @abc.abstractmethod + def RenameTable(self, request, timeout): + raise NotImplementedError() + RenameTable.future = None + @abc.abstractmethod + def CreateColumnFamily(self, request, timeout): + raise NotImplementedError() + CreateColumnFamily.future = None + @abc.abstractmethod + def UpdateColumnFamily(self, request, timeout): + raise NotImplementedError() + UpdateColumnFamily.future = None + @abc.abstractmethod + def DeleteColumnFamily(self, request, timeout): + raise NotImplementedError() + DeleteColumnFamily.future = None + +def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + request_deserializers = { + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.GetTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, + } + response_serializers = { + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + } + method_implementations = { + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): face_utilities.unary_unary_inline(servicer.CreateColumnFamily), + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): face_utilities.unary_unary_inline(servicer.CreateTable), + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): face_utilities.unary_unary_inline(servicer.DeleteColumnFamily), + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): face_utilities.unary_unary_inline(servicer.DeleteTable), + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): face_utilities.unary_unary_inline(servicer.GetTable), + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): face_utilities.unary_unary_inline(servicer.ListTables), + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): face_utilities.unary_unary_inline(servicer.RenameTable), + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): face_utilities.unary_unary_inline(servicer.UpdateColumnFamily), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + +def beta_create_BigtableTableService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_data_pb2 + import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 + import gcloud_bigtable._generated.empty_pb2 + request_serializers = { + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + } + response_deserializers = { + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, + } + cardinalities = { + 'CreateColumnFamily': cardinality.Cardinality.UNARY_UNARY, + 'CreateTable': cardinality.Cardinality.UNARY_UNARY, + 'DeleteColumnFamily': cardinality.Cardinality.UNARY_UNARY, + 'DeleteTable': cardinality.Cardinality.UNARY_UNARY, + 'GetTable': cardinality.Cardinality.UNARY_UNARY, + 'ListTables': cardinality.Cardinality.UNARY_UNARY, + 'RenameTable': cardinality.Cardinality.UNARY_UNARY, + 'UpdateColumnFamily': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'google.bigtable.admin.table.v1.BigtableTableService', cardinalities, options=stub_options) # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/duration_pb2.py b/gcloud_bigtable/_generated/duration_pb2.py index 7be5b3c..ddb81a7 100644 --- a/gcloud_bigtable/_generated/duration_pb2.py +++ b/gcloud_bigtable/_generated/duration_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/protobuf/duration.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/protobuf/duration.proto', package='google.protobuf', syntax='proto3', - serialized_pb=_b('\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\"*\n\x08\x44uration\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42)\n\x13\x63om.google.protobufB\rDurationProtoP\x01\xa0\x01\x01\x62\x06proto3') + serialized_pb=b'\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\"*\n\x08\x44uration\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42)\n\x13\x63om.google.protobufB\rDurationProtoP\x01\xa0\x01\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -74,8 +72,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\rDurationProtoP\001\240\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\rDurationProtoP\001\240\001\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/empty_pb2.py b/gcloud_bigtable/_generated/empty_pb2.py index 976dfb9..701a9a6 100644 --- a/gcloud_bigtable/_generated/empty_pb2.py +++ b/gcloud_bigtable/_generated/empty_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/protobuf/empty.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/protobuf/empty.proto', package='google.protobuf', syntax='proto3', - serialized_pb=_b('\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB#\n\x13\x63om.google.protobufB\nEmptyProtoP\x01\x62\x06proto3') + serialized_pb=b'\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB#\n\x13\x63om.google.protobufB\nEmptyProtoP\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -60,8 +58,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\nEmptyProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\nEmptyProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/http_pb2.py b/gcloud_bigtable/_generated/http_pb2.py index f755e4c..44885b8 100644 --- a/gcloud_bigtable/_generated/http_pb2.py +++ b/gcloud_bigtable/_generated/http_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/api/http.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/api/http.proto', package='google.api', syntax='proto3', - serialized_pb=_b('\n\x15google/api/http.proto\x12\ngoogle.api\"\xd8\x01\n\x08HttpRule\x12\r\n\x03get\x18\x02 \x01(\tH\x00\x12\r\n\x03put\x18\x03 \x01(\tH\x00\x12\x0e\n\x04post\x18\x04 \x01(\tH\x00\x12\x10\n\x06\x64\x65lete\x18\x05 \x01(\tH\x00\x12\x0f\n\x05patch\x18\x06 \x01(\tH\x00\x12/\n\x06\x63ustom\x18\x08 \x01(\x0b\x32\x1d.google.api.CustomHttpPatternH\x00\x12\x0c\n\x04\x62ody\x18\x07 \x01(\t\x12\x31\n\x13\x61\x64\x64itional_bindings\x18\x0b \x03(\x0b\x32\x14.google.api.HttpRuleB\t\n\x07pattern\"/\n\x11\x43ustomHttpPattern\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\tB\x1d\n\x0e\x63om.google.apiB\tHttpProtoP\x01\x62\x06proto3') + serialized_pb=b'\n\x15google/api/http.proto\x12\ngoogle.api\"\xd8\x01\n\x08HttpRule\x12\r\n\x03get\x18\x02 \x01(\tH\x00\x12\r\n\x03put\x18\x03 \x01(\tH\x00\x12\x0e\n\x04post\x18\x04 \x01(\tH\x00\x12\x10\n\x06\x64\x65lete\x18\x05 \x01(\tH\x00\x12\x0f\n\x05patch\x18\x06 \x01(\tH\x00\x12/\n\x06\x63ustom\x18\x08 \x01(\x0b\x32\x1d.google.api.CustomHttpPatternH\x00\x12\x0c\n\x04\x62ody\x18\x07 \x01(\t\x12\x31\n\x13\x61\x64\x64itional_bindings\x18\x0b \x03(\x0b\x32\x14.google.api.HttpRuleB\t\n\x07pattern\"/\n\x11\x43ustomHttpPattern\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\tB\x1d\n\x0e\x63om.google.apiB\tHttpProtoP\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -36,35 +34,35 @@ _descriptor.FieldDescriptor( name='get', full_name='google.api.HttpRule.get', index=0, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='put', full_name='google.api.HttpRule.put', index=1, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='post', full_name='google.api.HttpRule.post', index=2, number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='delete', full_name='google.api.HttpRule.delete', index=3, number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='patch', full_name='google.api.HttpRule.patch', index=4, number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -78,7 +76,7 @@ _descriptor.FieldDescriptor( name='body', full_name='google.api.HttpRule.body', index=6, number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -119,14 +117,14 @@ _descriptor.FieldDescriptor( name='kind', full_name='google.api.CustomHttpPattern.kind', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='path', full_name='google.api.CustomHttpPattern.path', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -185,8 +183,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.apiB\tHttpProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\tHttpProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/operations_pb2.py b/gcloud_bigtable/_generated/operations_pb2.py index b31048c..adbe718 100644 --- a/gcloud_bigtable/_generated/operations_pb2.py +++ b/gcloud_bigtable/_generated/operations_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/longrunning/operations.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -23,7 +21,7 @@ name='google/longrunning/operations.proto', package='google.longrunning', syntax='proto3', - serialized_pb=_b('\n#google/longrunning/operations.proto\x12\x12google.longrunning\x1a\x1cgoogle/api/annotations.proto\x1a\x19google/protobuf/any.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x17google/rpc/status.proto\"\xa8\x01\n\tOperation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x08metadata\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12#\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12(\n\x08response\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x42\x08\n\x06result\"#\n\x13GetOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\\\n\x15ListOperationsRequest\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"d\n\x16ListOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"&\n\x16\x43\x61ncelOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n\x16\x44\x65leteOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t2\x8c\x04\n\nOperations\x12x\n\x0cGetOperation\x12\'.google.longrunning.GetOperationRequest\x1a\x1d.google.longrunning.Operation\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v1/{name=operations/**}\x12\x86\x01\n\x0eListOperations\x12).google.longrunning.ListOperationsRequest\x1a*.google.longrunning.ListOperationsResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v1/{name=operations}\x12\x81\x01\n\x0f\x43\x61ncelOperation\x12*.google.longrunning.CancelOperationRequest\x1a\x16.google.protobuf.Empty\"*\x82\xd3\xe4\x93\x02$\"\x1f/v1/{name=operations/**}:cancel:\x01*\x12w\n\x0f\x44\x65leteOperation\x12*.google.longrunning.DeleteOperationRequest\x1a\x16.google.protobuf.Empty\" \x82\xd3\xe4\x93\x02\x1a*\x18/v1/{name=operations/**}B+\n\x16\x63om.google.longrunningB\x0fOperationsProtoP\x01\x62\x06proto3') + serialized_pb=b'\n#google/longrunning/operations.proto\x12\x12google.longrunning\x1a\x1cgoogle/api/annotations.proto\x1a\x19google/protobuf/any.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x17google/rpc/status.proto\"\xa8\x01\n\tOperation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x08metadata\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12#\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12(\n\x08response\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x42\x08\n\x06result\"#\n\x13GetOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\\\n\x15ListOperationsRequest\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"d\n\x16ListOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"&\n\x16\x43\x61ncelOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n\x16\x44\x65leteOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t2\x8c\x04\n\nOperations\x12x\n\x0cGetOperation\x12\'.google.longrunning.GetOperationRequest\x1a\x1d.google.longrunning.Operation\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v1/{name=operations/**}\x12\x86\x01\n\x0eListOperations\x12).google.longrunning.ListOperationsRequest\x1a*.google.longrunning.ListOperationsResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v1/{name=operations}\x12\x81\x01\n\x0f\x43\x61ncelOperation\x12*.google.longrunning.CancelOperationRequest\x1a\x16.google.protobuf.Empty\"*\x82\xd3\xe4\x93\x02$\"\x1f/v1/{name=operations/**}:cancel:\x01*\x12w\n\x0f\x44\x65leteOperation\x12*.google.longrunning.DeleteOperationRequest\x1a\x16.google.protobuf.Empty\" \x82\xd3\xe4\x93\x02\x1a*\x18/v1/{name=operations/**}B+\n\x16\x63om.google.longrunningB\x0fOperationsProtoP\x01\x62\x06proto3' , dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_any__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,google_dot_rpc_dot_status__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -41,7 +39,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.longrunning.Operation.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -103,7 +101,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.longrunning.GetOperationRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -134,14 +132,14 @@ _descriptor.FieldDescriptor( name='name', full_name='google.longrunning.ListOperationsRequest.name', index=0, number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( name='filter', full_name='google.longrunning.ListOperationsRequest.filter', index=1, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -155,7 +153,7 @@ _descriptor.FieldDescriptor( name='page_token', full_name='google.longrunning.ListOperationsRequest.page_token', index=3, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -193,7 +191,7 @@ _descriptor.FieldDescriptor( name='next_page_token', full_name='google.longrunning.ListOperationsResponse.next_page_token', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -224,7 +222,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.longrunning.CancelOperationRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -255,7 +253,7 @@ _descriptor.FieldDescriptor( name='name', full_name='google.longrunning.DeleteOperationRequest.name', index=0, number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -336,10 +334,13 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.longrunningB\017OperationsProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.longrunningB\017OperationsProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities class EarlyAdopterOperationsServicer(object): """""" __metaclass__ = abc.ABCMeta @@ -393,28 +394,28 @@ def early_adopter_create_Operations_server(servicer, port, private_key=None, cer import gcloud_bigtable._generated.operations_pb2 import gcloud_bigtable._generated.empty_pb2 method_service_descriptions = { - "CancelOperation": utilities.unary_unary_service_description( + "CancelOperation": alpha_utilities.unary_unary_service_description( servicer.CancelOperation, gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "DeleteOperation": utilities.unary_unary_service_description( + "DeleteOperation": alpha_utilities.unary_unary_service_description( servicer.DeleteOperation, gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.FromString, gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ), - "GetOperation": utilities.unary_unary_service_description( + "GetOperation": alpha_utilities.unary_unary_service_description( servicer.GetOperation, gcloud_bigtable._generated.operations_pb2.GetOperationRequest.FromString, gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, ), - "ListOperations": utilities.unary_unary_service_description( + "ListOperations": alpha_utilities.unary_unary_service_description( servicer.ListOperations, gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.FromString, gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.SerializeToString, ), } - return implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) + return early_adopter_implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): import gcloud_bigtable._generated.operations_pb2 import gcloud_bigtable._generated.operations_pb2 @@ -425,22 +426,118 @@ def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, import gcloud_bigtable._generated.operations_pb2 import gcloud_bigtable._generated.empty_pb2 method_invocation_descriptions = { - "CancelOperation": utilities.unary_unary_invocation_description( + "CancelOperation": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "DeleteOperation": utilities.unary_unary_invocation_description( + "DeleteOperation": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.SerializeToString, gcloud_bigtable._generated.empty_pb2.Empty.FromString, ), - "GetOperation": utilities.unary_unary_invocation_description( + "GetOperation": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.operations_pb2.GetOperationRequest.SerializeToString, gcloud_bigtable._generated.operations_pb2.Operation.FromString, ), - "ListOperations": utilities.unary_unary_invocation_description( + "ListOperations": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.SerializeToString, gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.FromString, ), } - return implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + return early_adopter_implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + +class BetaOperationsServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ListOperations(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CancelOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteOperation(self, request, context): + raise NotImplementedError() + +class BetaOperationsStub(object): + """The interface to which stubs will conform.""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request, timeout): + raise NotImplementedError() + GetOperation.future = None + @abc.abstractmethod + def ListOperations(self, request, timeout): + raise NotImplementedError() + ListOperations.future = None + @abc.abstractmethod + def CancelOperation(self, request, timeout): + raise NotImplementedError() + CancelOperation.future = None + @abc.abstractmethod + def DeleteOperation(self, request, timeout): + raise NotImplementedError() + DeleteOperation.future = None + +def beta_create_Operations_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.empty_pb2 + request_deserializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.FromString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.FromString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.FromString, + } + response_serializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, + } + method_implementations = { + ('google.longrunning.Operations', 'CancelOperation'): face_utilities.unary_unary_inline(servicer.CancelOperation), + ('google.longrunning.Operations', 'DeleteOperation'): face_utilities.unary_unary_inline(servicer.DeleteOperation), + ('google.longrunning.Operations', 'GetOperation'): face_utilities.unary_unary_inline(servicer.GetOperation), + ('google.longrunning.Operations', 'ListOperations'): face_utilities.unary_unary_inline(servicer.ListOperations), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + +def beta_create_Operations_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.empty_pb2 + import gcloud_bigtable._generated.operations_pb2 + import gcloud_bigtable._generated.empty_pb2 + request_serializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, + } + response_deserializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.FromString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.FromString, + } + cardinalities = { + 'CancelOperation': cardinality.Cardinality.UNARY_UNARY, + 'DeleteOperation': cardinality.Cardinality.UNARY_UNARY, + 'GetOperation': cardinality.Cardinality.UNARY_UNARY, + 'ListOperations': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'google.longrunning.Operations', cardinalities, options=stub_options) # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/status_pb2.py b/gcloud_bigtable/_generated/status_pb2.py index 643f286..a715af1 100644 --- a/gcloud_bigtable/_generated/status_pb2.py +++ b/gcloud_bigtable/_generated/status_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/rpc/status.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -20,7 +18,7 @@ name='google/rpc/status.proto', package='google.rpc', syntax='proto3', - serialized_pb=_b('\n\x17google/rpc/status.proto\x12\ngoogle.rpc\x1a\x19google/protobuf/any.proto\"N\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12%\n\x07\x64\x65tails\x18\x03 \x03(\x0b\x32\x14.google.protobuf.AnyB\x1f\n\x0e\x63om.google.rpcB\x0bStatusProtoP\x01\x62\x06proto3') + serialized_pb=b'\n\x17google/rpc/status.proto\x12\ngoogle.rpc\x1a\x19google/protobuf/any.proto\"N\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12%\n\x07\x64\x65tails\x18\x03 \x03(\x0b\x32\x14.google.protobuf.AnyB\x1f\n\x0e\x63om.google.rpcB\x0bStatusProtoP\x01\x62\x06proto3' , dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -45,7 +43,7 @@ _descriptor.FieldDescriptor( name='message', full_name='google.rpc.Status.message', index=1, number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -84,8 +82,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.rpcB\013StatusProtoP\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.rpcB\013StatusProtoP\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/timestamp_pb2.py b/gcloud_bigtable/_generated/timestamp_pb2.py index 6baa9b4..66a6cf6 100644 --- a/gcloud_bigtable/_generated/timestamp_pb2.py +++ b/gcloud_bigtable/_generated/timestamp_pb2.py @@ -1,8 +1,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: google/protobuf/timestamp.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -19,7 +17,7 @@ name='google/protobuf/timestamp.proto', package='google.protobuf', syntax='proto3', - serialized_pb=_b('\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\"+\n\tTimestamp\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42*\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01\xa0\x01\x01\x62\x06proto3') + serialized_pb=b'\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\"+\n\tTimestamp\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42*\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01\xa0\x01\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -74,8 +72,11 @@ DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\016TimestampProtoP\001\240\001\001')) +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\016TimestampProtoP\001\240\001\001') import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) From ef249c56095ace25fd0e747115d5757d2140dbc8 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 23 Nov 2015 09:52:32 -0800 Subject: [PATCH 02/65] Adding fake test modules for gRPC. --- _fake_grpc/grpc/beta/__init__.py | 13 +++++++++++++ _fake_grpc/grpc/beta/implementations.py | 13 +++++++++++++ _fake_grpc/grpc/framework/common/__init__.py | 13 +++++++++++++ _fake_grpc/grpc/framework/common/cardinality.py | 13 +++++++++++++ _fake_grpc/grpc/framework/interfaces/__init__.py | 13 +++++++++++++ .../grpc/framework/interfaces/face/__init__.py | 13 +++++++++++++ .../grpc/framework/interfaces/face/utilities.py | 13 +++++++++++++ 7 files changed, 91 insertions(+) create mode 100644 _fake_grpc/grpc/beta/__init__.py create mode 100644 _fake_grpc/grpc/beta/implementations.py create mode 100644 _fake_grpc/grpc/framework/common/__init__.py create mode 100644 _fake_grpc/grpc/framework/common/cardinality.py create mode 100644 _fake_grpc/grpc/framework/interfaces/__init__.py create mode 100644 _fake_grpc/grpc/framework/interfaces/face/__init__.py create mode 100644 _fake_grpc/grpc/framework/interfaces/face/utilities.py diff --git a/_fake_grpc/grpc/beta/__init__.py b/_fake_grpc/grpc/beta/__init__.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/beta/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/beta/implementations.py b/_fake_grpc/grpc/beta/implementations.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/beta/implementations.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/common/__init__.py b/_fake_grpc/grpc/framework/common/__init__.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/common/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/common/cardinality.py b/_fake_grpc/grpc/framework/common/cardinality.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/common/cardinality.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/__init__.py b/_fake_grpc/grpc/framework/interfaces/__init__.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/interfaces/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/__init__.py b/_fake_grpc/grpc/framework/interfaces/face/__init__.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/interfaces/face/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/utilities.py b/_fake_grpc/grpc/framework/interfaces/face/utilities.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/interfaces/face/utilities.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 9ac3de58020ecf33268d68e63e13e87596b25432 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 08:56:06 -0800 Subject: [PATCH 03/65] Dropping dependence on Homebrew/Linuxbrew. Also upgrading version of grpcio in system tests. The library supports backwards compatibility, so the code runs fine without change. --- Makefile | 4 +--- scripts/nose_with_env.sh | 23 ----------------------- scripts/sphinx_with_env.sh | 23 ----------------------- tox.ini | 12 ++++++------ 4 files changed, 7 insertions(+), 55 deletions(-) delete mode 100755 scripts/nose_with_env.sh delete mode 100755 scripts/sphinx_with_env.sh diff --git a/Makefile b/Makefile index 7a8b6d8..11cb68b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ GENERATED_DIR=$(shell pwd)/generated_python FINAL_DIR=gcloud_bigtable/_generated -BREW_PREFIX=$(shell brew --prefix) -LD_LIBRARY_PATH=$(BREW_PREFIX)/lib -GRPC_PLUGIN=$(BREW_PREFIX)/bin/grpc_python_plugin +GRPC_PLUGIN=grpc_python_plugin help: @echo 'Makefile for a gcloud-python-bigtable ' diff --git a/scripts/nose_with_env.sh b/scripts/nose_with_env.sh deleted file mode 100755 index d83e01d..0000000 --- a/scripts/nose_with_env.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -if [[ -n "$(which brew)" ]]; then - export LD_LIBRARY_PATH=$(brew --prefix)/lib -fi - -nosetests ${@} diff --git a/scripts/sphinx_with_env.sh b/scripts/sphinx_with_env.sh deleted file mode 100755 index f2ff46f..0000000 --- a/scripts/sphinx_with_env.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -if [[ -n "$(which brew)" ]]; then - export LD_LIBRARY_PATH=$(brew --prefix)/lib -fi - -sphinx-build ${@} diff --git a/tox.ini b/tox.ini index 8d64601..ee054c5 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = [testenv] commands = - {toxinidir}/scripts/nose_with_env.sh + nosetests deps = nose unittest2 @@ -15,7 +15,7 @@ setenv = basepython = python2.7 commands = - {toxinidir}/scripts/nose_with_env.sh --with-xunit --with-xcoverage --cover-package=gcloud_bigtable --nocapture --cover-erase --cover-tests --cover-branches --cover-min-percentage=100 + nosetests --with-xunit --with-xcoverage --cover-package=gcloud_bigtable --nocapture --cover-erase --cover-tests --cover-branches --cover-min-percentage=100 deps = {[testenv]deps} coverage @@ -41,10 +41,10 @@ passenv = GCLOUD_* basepython = python2.7 commands = - {toxinidir}/scripts/nose_with_env.sh -v {toxinidir}/system_tests/run.py - {toxinidir}/scripts/nose_with_env.sh -v {toxinidir}/system_tests/run_happybase.py + nosetests -v {toxinidir}/system_tests/run.py + nosetests -v {toxinidir}/system_tests/run_happybase.py deps = - grpcio==0.10.0a0 + grpcio==0.11.0b1 nose unittest2 passenv = GOOGLE_* GCLOUD_* GRPC_TRACE @@ -55,7 +55,7 @@ basepython = python2.7 commands = python -c "import shutil; shutil.rmtree('docs/_build', ignore_errors=True)" - {toxinidir}/scripts/sphinx_with_env.sh -W -b html -d docs/_build/doctrees docs docs/_build/html + sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html deps = Sphinx passenv = SPHINX_RELEASE From 77f3451c50281618fbf82f12ab8631f39e5ef405 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 10:11:11 -0800 Subject: [PATCH 04/65] Switching operations stub from alpha to beta impl. In order to actually execute the code, some rewritten imports had to be dealt with (they were referenced by the original import name in a function call). Ran some `sed` commands to avoid adding extra hacks to `rewrite_imports.py`: git grep -l ' google\.longrunning\.' | xargs sed -i 's/ google\.longrunning\./ gcloud_bigtable\._generated\./g' git grep -l ' google\.protobuf\.empty_pb2\.' | xargs sed -i 's/ google\.protobuf\.empty_pb2\./ gcloud_bigtable\._generated\.empty_pb2\./g' --- .../bigtable_cluster_service_pb2.py | 8 +-- .../_generated/bigtable_service_pb2.py | 4 +- .../_generated/bigtable_table_service_pb2.py | 12 ++--- gcloud_bigtable/_generated/operations_pb2.py | 32 ++++++------ gcloud_bigtable/_grpc_mocks.py | 3 ++ gcloud_bigtable/_helpers.py | 37 +++++++++++++- gcloud_bigtable/client.py | 9 ++-- gcloud_bigtable/cluster.py | 5 +- gcloud_bigtable/test__helpers.py | 49 +++++++++++++++++++ gcloud_bigtable/test_client.py | 5 +- 10 files changed, 126 insertions(+), 38 deletions(-) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py index 43be17d..b5d5100 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py @@ -286,11 +286,11 @@ def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=Non } response_serializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, } method_implementations = { @@ -331,11 +331,11 @@ def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transfo } response_deserializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, } cardinalities = { diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index bd0900f..01724ce 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -223,7 +223,7 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa } response_serializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, @@ -258,7 +258,7 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No } response_deserializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.FromString, ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.FromString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py index 4567385..e17e1da 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py @@ -316,11 +316,11 @@ def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, response_serializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, } method_implementations = { @@ -366,11 +366,11 @@ def beta_create_BigtableTableService_stub(channel, host=None, metadata_transform response_deserializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, } cardinalities = { diff --git a/gcloud_bigtable/_generated/operations_pb2.py b/gcloud_bigtable/_generated/operations_pb2.py index adbe718..069453b 100644 --- a/gcloud_bigtable/_generated/operations_pb2.py +++ b/gcloud_bigtable/_generated/operations_pb2.py @@ -491,16 +491,16 @@ def beta_create_Operations_server(servicer, pool=None, pool_size=None, default_t import gcloud_bigtable._generated.operations_pb2 import gcloud_bigtable._generated.empty_pb2 request_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.FromString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.FromString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.FromString, + ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.FromString, + ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.GetOperationRequest.FromString, + ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.FromString, } response_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, + ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.SerializeToString, } method_implementations = { ('google.longrunning.Operations', 'CancelOperation'): face_utilities.unary_unary_inline(servicer.CancelOperation), @@ -521,16 +521,16 @@ def beta_create_Operations_stub(channel, host=None, metadata_transformer=None, p import gcloud_bigtable._generated.operations_pb2 import gcloud_bigtable._generated.empty_pb2 request_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, + ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.GetOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.SerializeToString, } response_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.FromString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.FromString, + ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.Operation.FromString, + ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.FromString, } cardinalities = { 'CancelOperation': cardinality.Cardinality.UNARY_UNARY, diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py index b8c2fe2..27b2e47 100644 --- a/gcloud_bigtable/_grpc_mocks.py +++ b/gcloud_bigtable/_grpc_mocks.py @@ -29,6 +29,9 @@ def result(self): class MethodMock(object): """Mock for :class:`grpc.framework.alpha._reexport._UnaryUnarySyncAsync`. + Can also be used for beta equivalent: + :class:`grpc.framework.crust.implementations._UnaryUnaryMultiCallable` + May need to be callable and needs to (in our use) have an ``async`` method. """ diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 806cbd2..d0e6499 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -23,6 +23,8 @@ import pytz import six +from grpc.beta import implementations + from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 @@ -272,7 +274,7 @@ def get_certs(): def make_stub(client, stub_factory, host, port): - """Makes a stub for the an API. + """Makes a stub for an RPC service. :type client: :class:`.client.Client` :param client: The client that owns the cluster. Provides authorization and @@ -299,6 +301,39 @@ def make_stub(client, stub_factory, host, port): root_certificates=get_certs()) +def make_beta_stub(client, stub_factory, host, port): + """Makes a (beta) stub for an RPC service. + + Uses the beta implementation of gRPC. + + :type client: :class:`.client.Client` + :param client: The client that owns the cluster. Provides authorization and + user agent. + + :type stub_factory: callable + :param stub_factory: A factory which will create a gRPC stub for + a given service. + + :type host: str + :param host: The host for the service. + + :type port: int + :param port: The port for the service. + + :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :returns: The stub object used to make gRPC requests to the + Data API. + """ + root_certificates = get_certs() + client_credentials = implementations.ssl_client_credentials( + root_certificates, private_key=None, certificate_chain=None) + channel = implementations.secure_channel( + host, port, client_credentials) + custom_metadata_transformer = MetadataTransformer(client) + return stub_factory(channel, host=host, + metadata_transformer=custom_metadata_transformer) + + def _parse_family_pb(family_pb): """Parses a Family protobuf into a dictionary. diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index fef9300..52c0a5a 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -49,6 +49,7 @@ from gcloud_bigtable._generated import bigtable_table_service_pb2 from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._helpers import make_stub +from gcloud_bigtable._helpers import make_beta_stub from gcloud_bigtable.cluster import Cluster @@ -73,7 +74,7 @@ DATA_API_PORT = 443 """Data API request port.""" -OPERATIONS_STUB_FACTORY = operations_pb2.early_adopter_create_Operations_stub +OPERATIONS_STUB_FACTORY = operations_pb2.beta_create_Operations_stub ADMIN_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.admin' """Scope for interacting with the Cluster Admin and Table Admin APIs.""" @@ -467,11 +468,11 @@ def _make_operations_stub(self): These are for long-running operations of the Cluster Admin API, hence the host and port matching. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return make_beta_stub(self, OPERATIONS_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_table_stub(self): """Creates gRPC stub to make requests to the Table Admin API. diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 00a9c72..fd22041 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -294,10 +294,9 @@ def operation_finished(self, timeout_seconds=None): '/operations/%d' % (self._operation_id,)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._operations_stub.GetOperation.async( - request_pb, timeout_seconds) # We expact a `._generated.operations_pb2.Operation`. - operation_pb = response.result() + operation_pb = self.client._operations_stub.GetOperation( + request_pb, timeout_seconds) if operation_pb.done: self._operation_type = None diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index ef98615..6039a53 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -444,6 +444,55 @@ def test_it(self): self.assertEqual(client._called, []) +class Test_make_beta_stub(unittest2.TestCase): + + def _callFUT(self, credentials, stub_factory, host, port): + from gcloud_bigtable._helpers import make_beta_stub + return make_beta_stub(credentials, stub_factory, host, port) + + def test_it(self): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _helpers as MUT + + mock_result = object() + custom_factory = _MockCalled(mock_result) + transformed = object() + transformer = _MockCalled(transformed) + + client_creds = object() + channel = object() + implementations_mod = _MockWithAttachedMethods(client_creds, channel) + + host = 'HOST' + port = 1025 + certs = 'FOOBAR' + client = _MockWithAttachedMethods() + with _Monkey(MUT, get_certs=lambda: certs, + implementations=implementations_mod, + MetadataTransformer=transformer): + result = self._callFUT(client, custom_factory, host, port) + + self.assertTrue(result is mock_result) + custom_factory.check_called( + self, + [(channel,)], + [{ + 'host': host, + 'metadata_transformer': transformed, + }], + ) + transformer.check_called(self, [(client,)]) + self.assertEqual(client._called, []) + # Check what was called on the module. + self.assertEqual(implementations_mod._called, [ + ('ssl_client_credentials', (certs,), + {'private_key': None, 'certificate_chain': None}), + ('secure_channel', (host, port, client_creds), {}), + ]) + + class Test__parse_family_pb(unittest2.TestCase): def _callFUT(self, family_pb): diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index 324914c..8086a9b 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -611,7 +611,7 @@ def test__make_operations_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, make_beta_stub=mock_make_stub): result = client._make_operations_stub() self.assertTrue(result is expected_result) @@ -677,7 +677,8 @@ def _start_method_helper(self, admin): credentials=credentials, admin=admin) stub = _FakeStub() mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub, + make_beta_stub=mock_make_stub): client.start() self.assertTrue(client._data_stub_internal is stub) From 2338ce319f982ee7a277e33e52a6304b451fd527 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 10:24:19 -0800 Subject: [PATCH 05/65] Switching cluster API stub from alpha to beta impl. Again ran some `sed` commands to avoid adding extra hacks to `rewrite_imports.py`: git grep -l ' google\.bigtable\.admin\.cluster\.v1\.' | xargs sed -i 's/ google\.bigtable\.admin\.cluster\.v1\./ gcloud_bigtable\._generated\./g' --- .../bigtable_cluster_service_pb2.py | 48 +++++++++---------- gcloud_bigtable/client.py | 18 ++++--- gcloud_bigtable/cluster.py | 24 ++++------ gcloud_bigtable/test_client.py | 2 +- 4 files changed, 42 insertions(+), 50 deletions(-) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py index b5d5100..7a5bad8 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py @@ -276,22 +276,22 @@ def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=Non import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 import gcloud_bigtable._generated.operations_pb2 request_deserializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, } response_serializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, } method_implementations = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): face_utilities.unary_unary_inline(servicer.CreateCluster), @@ -321,22 +321,22 @@ def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transfo import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 import gcloud_bigtable._generated.operations_pb2 request_serializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, } response_deserializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): google.bigtable.admin.cluster.v1.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): google.bigtable.admin.cluster.v1.bigtable_cluster_data_pb2.Cluster.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, } cardinalities = { 'CreateCluster': cardinality.Cardinality.UNARY_UNARY, diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 52c0a5a..f0a6f06 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -60,8 +60,8 @@ TABLE_ADMIN_PORT = 443 """Table Admin API request port.""" -CLUSTER_STUB_FACTORY = (bigtable_cluster_service_pb2. - early_adopter_create_BigtableClusterService_stub) +CLUSTER_STUB_FACTORY = ( + bigtable_cluster_service_pb2.beta_create_BigtableClusterService_stub) CLUSTER_ADMIN_HOST = 'bigtableclusteradmin.googleapis.com' """Cluster Admin API request host.""" CLUSTER_ADMIN_PORT = 443 @@ -459,8 +459,8 @@ def _make_cluster_stub(self): :rtype: :class:`grpc.early_adopter.implementations._Stub` :returns: A gRPC stub object. """ - return make_stub(self, CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return make_beta_stub(self, CLUSTER_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_operations_stub(self): """Creates gRPC stub to make requests to the Operations API. @@ -573,10 +573,9 @@ def list_zones(self, timeout_seconds=None): """ request_pb = messages_pb2.ListZonesRequest(name=self.project_name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self._cluster_stub.ListZones.async(request_pb, - timeout_seconds) # We expect a `.messages_pb2.ListZonesResponse` - list_zones_response = response.result() + list_zones_response = self._cluster_stub.ListZones(request_pb, + timeout_seconds) result = [] for zone in list_zones_response.zones: @@ -600,10 +599,9 @@ def list_clusters(self, timeout_seconds=None): """ request_pb = messages_pb2.ListClustersRequest(name=self.project_name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self._cluster_stub.ListClusters.async(request_pb, - timeout_seconds) # We expect a `.messages_pb2.ListClustersResponse` - list_clusters_response = response.result() + list_clusters_response = self._cluster_stub.ListClusters( + request_pb, timeout_seconds) failed_zones = [zone.display_name for zone in list_clusters_response.failed_zones] diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index fd22041..c282742 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -265,10 +265,9 @@ def reload(self, timeout_seconds=None): """ request_pb = messages_pb2.GetClusterRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.GetCluster.async(request_pb, - timeout_seconds) # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = response.result() + cluster_pb = self.client._cluster_stub.GetCluster(request_pb, + timeout_seconds) # NOTE: _update_from_pb does not check that the project, zone and # cluster ID on the response match the request. @@ -330,10 +329,9 @@ def create(self, timeout_seconds=None): """ request_pb = _prepare_create_request(self) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.CreateCluster.async( - request_pb, timeout_seconds) # We expect an `operations_pb2.Operation`. - cluster_pb = response.result() + cluster_pb = self.client._cluster_stub.CreateCluster( + request_pb, timeout_seconds) self._operation_type = 'create' self._operation_id, self._operation_begin = _process_operation( @@ -365,10 +363,9 @@ def update(self, timeout_seconds=None): serve_nodes=self.serve_nodes, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.UpdateCluster.async( - request_pb, timeout_seconds) # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = response.result() + cluster_pb = self.client._cluster_stub.UpdateCluster( + request_pb, timeout_seconds) self._operation_type = 'update' self._operation_id, self._operation_begin = _process_operation( @@ -384,10 +381,8 @@ def delete(self, timeout_seconds=None): """ request_pb = messages_pb2.DeleteClusterRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.DeleteCluster.async( - request_pb, timeout_seconds) # We expect a `._generated.empty_pb2.Empty` - response.result() + self.client._cluster_stub.DeleteCluster(request_pb, timeout_seconds) def undelete(self, timeout_seconds=None): """Undelete this cluster. @@ -399,10 +394,9 @@ def undelete(self, timeout_seconds=None): """ request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.UndeleteCluster.async( - request_pb, timeout_seconds) # We expect a `._generated.operations_pb2.Operation` - operation_pb2 = response.result() + operation_pb2 = self.client._cluster_stub.UndeleteCluster( + request_pb, timeout_seconds) self._operation_type = 'undelete' self._operation_id, self._operation_begin = _process_operation( diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index 8086a9b..5ad7dd2 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -583,7 +583,7 @@ def test__make_cluster_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, make_beta_stub=mock_make_stub): result = client._make_cluster_stub() self.assertTrue(result is expected_result) From edf491aaa57e8a434cd38aa35cb599f1fda17213 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 15:45:04 -0800 Subject: [PATCH 06/65] Switching table API stub from alpha to beta impl. Again ran some `sed` commands to avoid adding extra hacks to `rewrite_imports.py`: git grep -l ' google\.bigtable\.admin\.table\.v1\.' | xargs sed -i 's/ google\.bigtable\.admin\.table\.v1\./ gcloud_bigtable\._generated\./g' --- .../_generated/bigtable_table_service_pb2.py | 52 +++++++++---------- gcloud_bigtable/client.py | 12 ++--- gcloud_bigtable/cluster.py | 5 +- gcloud_bigtable/column_family.py | 15 +++--- gcloud_bigtable/table.py | 17 ++---- gcloud_bigtable/test_client.py | 2 +- 6 files changed, 46 insertions(+), 57 deletions(-) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py index e17e1da..50c51a9 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py @@ -304,24 +304,24 @@ def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.empty_pb2 request_deserializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.GetTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, } response_serializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, } method_implementations = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): face_utilities.unary_unary_inline(servicer.CreateColumnFamily), @@ -354,24 +354,24 @@ def beta_create_BigtableTableService_stub(channel, host=None, metadata_transform import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.empty_pb2 request_serializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, } response_deserializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.Table.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): google.bigtable.admin.table.v1.bigtable_table_data_pb2.ColumnFamily.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, } cardinalities = { 'CreateColumnFamily': cardinality.Cardinality.UNARY_UNARY, diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index f0a6f06..abdc19d 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -53,8 +53,8 @@ from gcloud_bigtable.cluster import Cluster -TABLE_STUB_FACTORY = (bigtable_table_service_pb2. - early_adopter_create_BigtableTableService_stub) +TABLE_STUB_FACTORY = ( + bigtable_table_service_pb2.beta_create_BigtableTableService_stub) TABLE_ADMIN_HOST = 'bigtabletableadmin.googleapis.com' """Table Admin API request host.""" TABLE_ADMIN_PORT = 443 @@ -456,7 +456,7 @@ def _make_data_stub(self): def _make_cluster_stub(self): """Creates gRPC stub to make requests to the Cluster Admin API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ return make_beta_stub(self, CLUSTER_STUB_FACTORY, @@ -477,11 +477,11 @@ def _make_operations_stub(self): def _make_table_stub(self): """Creates gRPC stub to make requests to the Table Admin API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) + return make_beta_stub(self, TABLE_STUB_FACTORY, + TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) def is_started(self): """Check if the client has been started. diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index c282742..9a15611 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -417,10 +417,9 @@ def list_tables(self, timeout_seconds=None): """ request_pb = table_messages_pb2.ListTablesRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.ListTables.async(request_pb, - timeout_seconds) # We expect a `table_messages_pb2.ListTablesResponse` - table_list_pb = response.result() + table_list_pb = self.client._table_stub.ListTables(request_pb, + timeout_seconds) result = [] for table_pb in table_list_pb.tables: diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index 58b3973..c85d2d5 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -255,10 +255,9 @@ def create(self, timeout_seconds=None): column_family=column_family, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.CreateColumnFamily.async( - request_pb, timeout_seconds) # We expect a `.data_pb2.ColumnFamily` - response.result() + self.client._table_stub.CreateColumnFamily(request_pb, + timeout_seconds) def update(self, timeout_seconds=None): """Update this column family. @@ -282,10 +281,9 @@ def update(self, timeout_seconds=None): request_kwargs['gc_rule'] = self.gc_rule.to_pb() request_pb = data_pb2.ColumnFamily(**request_kwargs) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.UpdateColumnFamily.async( - request_pb, timeout_seconds) # We expect a `.data_pb2.ColumnFamily` - response.result() + self.client._table_stub.UpdateColumnFamily(request_pb, + timeout_seconds) def delete(self, timeout_seconds=None): """Delete this column family. @@ -297,10 +295,9 @@ def delete(self, timeout_seconds=None): """ request_pb = messages_pb2.DeleteColumnFamilyRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.DeleteColumnFamily.async( - request_pb, timeout_seconds) # We expect a `._generated.empty_pb2.Empty` - response.result() + self.client._table_stub.DeleteColumnFamily(request_pb, + timeout_seconds) def _gc_rule_from_pb(gc_rule_pb): diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index b5fbec1..d471e6b 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -189,10 +189,8 @@ def create(self, initial_split_keys=None, timeout_seconds=None): table_id=self.table_id, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.CreateTable.async(request_pb, - timeout_seconds) # We expect a `._generated.bigtable_table_data_pb2.Table` - response.result() + self.client._table_stub.CreateTable(request_pb, timeout_seconds) def rename(self, new_table_id, timeout_seconds=None): """Rename this table. @@ -223,10 +221,8 @@ def rename(self, new_table_id, timeout_seconds=None): new_id=new_table_id, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.RenameTable.async(request_pb, - timeout_seconds) # We expect a `._generated.empty_pb2.Empty` - response.result() + self.client._table_stub.RenameTable(request_pb, timeout_seconds) self.table_id = new_table_id @@ -239,10 +235,8 @@ def delete(self, timeout_seconds=None): """ request_pb = messages_pb2.DeleteTableRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.DeleteTable.async(request_pb, - timeout_seconds) # We expect a `._generated.empty_pb2.Empty` - response.result() + self.client._table_stub.DeleteTable(request_pb, timeout_seconds) def list_column_families(self, timeout_seconds=None): """Check if this table exists. @@ -260,10 +254,9 @@ def list_column_families(self, timeout_seconds=None): """ request_pb = messages_pb2.GetTableRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.GetTable.async(request_pb, - timeout_seconds) # We expect a `._generated.bigtable_table_data_pb2.Table` - table_pb = response.result() + table_pb = self.client._table_stub.GetTable(request_pb, + timeout_seconds) result = {} for column_family_id, value_pb in table_pb.column_families.items(): diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index 5ad7dd2..8f6b57b 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -639,7 +639,7 @@ def test__make_table_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, make_beta_stub=mock_make_stub): result = client._make_table_stub() self.assertTrue(result is expected_result) From dca3fb26cfba320dae380b14f5834b9a1ca5061e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 15:49:59 -0800 Subject: [PATCH 07/65] Switching data API stub from alpha to beta impl. Again ran some `sed` commands to avoid adding extra hacks to `rewrite_imports.py`: git grep -l ' google\.bigtable\.v1\.' | xargs sed -i 's/ google\.bigtable\.v1\./ gcloud_bigtable\._generated\./g' --- .../_generated/bigtable_service_pb2.py | 36 +++++++++---------- gcloud_bigtable/_grpc_mocks.py | 18 ---------- gcloud_bigtable/client.py | 8 ++--- gcloud_bigtable/row.py | 16 ++++----- gcloud_bigtable/test_client.py | 5 ++- 5 files changed, 29 insertions(+), 54 deletions(-) diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index 01724ce..3b0163c 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -215,18 +215,18 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 request_deserializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.MutateRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsRequest.FromString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, } response_serializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, } method_implementations = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): face_utilities.unary_unary_inline(servicer.CheckAndMutateRow), @@ -250,18 +250,18 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 request_serializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, } response_deserializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): google.bigtable.v1.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, + ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): google.bigtable.v1.bigtable_data_pb2.Row.FromString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): google.bigtable.v1.bigtable_service_messages_pb2.ReadRowsResponse.FromString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): google.bigtable.v1.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, + ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, + ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, + ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, } cardinalities = { 'CheckAndMutateRow': cardinality.Cardinality.UNARY_UNARY, diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py index 27b2e47..06a6cc9 100644 --- a/gcloud_bigtable/_grpc_mocks.py +++ b/gcloud_bigtable/_grpc_mocks.py @@ -15,17 +15,6 @@ """Testing mocks for grpc.""" -class AsyncResult(object): - """Result returned from a ``MethodMock.async`` call.""" - - def __init__(self, result): - self._result = result - - def result(self): - """Result method on an asyc object.""" - return self._result - - class MethodMock(object): """Mock for :class:`grpc.framework.alpha._reexport._UnaryUnarySyncAsync`. @@ -40,13 +29,6 @@ def __init__(self, name, factory): self._name = name self._factory = factory - def async(self, *args, **kwargs): - """Async method meant to mock a gRPC stub request.""" - self._factory.method_calls.append((self._name, args, kwargs)) - curr_result = self._factory.results[0] - self._factory.results = self._factory.results[1:] - return AsyncResult(curr_result) - def __call__(self, *args, **kwargs): """Sync method meant to mock a gRPC stub request.""" self._factory.method_calls.append((self._name, args, kwargs)) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index abdc19d..672b7a8 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -48,7 +48,6 @@ from gcloud_bigtable._generated import bigtable_service_pb2 from gcloud_bigtable._generated import bigtable_table_service_pb2 from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import make_stub from gcloud_bigtable._helpers import make_beta_stub from gcloud_bigtable.cluster import Cluster @@ -67,8 +66,7 @@ CLUSTER_ADMIN_PORT = 443 """Cluster Admin API request port.""" -DATA_STUB_FACTORY = (bigtable_service_pb2. - early_adopter_create_BigtableService_stub) +DATA_STUB_FACTORY = bigtable_service_pb2.beta_create_BigtableService_stub DATA_API_HOST = 'bigtable.googleapis.com' """Data API request host.""" DATA_API_PORT = 443 @@ -450,8 +448,8 @@ def _make_data_stub(self): :rtype: :class:`grpc.early_adopter.implementations._Stub` :returns: A gRPC stub object. """ - return make_stub(self, DATA_STUB_FACTORY, - DATA_API_HOST, DATA_API_PORT) + return make_beta_stub(self, DATA_STUB_FACTORY, + DATA_API_HOST, DATA_API_PORT) def _make_cluster_stub(self): """Creates gRPC stub to make requests to the Cluster Admin API. diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 5d26c49..35ed9c4 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -416,10 +416,8 @@ def _commit_mutate(self, timeout_seconds=None): mutations=mutations_list, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.MutateRow.async(request_pb, - timeout_seconds) # We expect a `._generated.empty_pb2.Empty`. - response.result() + self.client._data_stub.MutateRow(request_pb, timeout_seconds) def _commit_check_and_mutate(self, timeout_seconds=None): """Makes a ``CheckAndMutateRow`` API request. @@ -458,11 +456,10 @@ def _commit_check_and_mutate(self, timeout_seconds=None): false_mutations=false_mutations, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.CheckAndMutateRow.async( - request_pb, timeout_seconds) # We expect a `.messages_pb2.CheckAndMutateRowResponse` - check_and_mutate_row_response = response.result() - return check_and_mutate_row_response.predicate_matched + resp = self.client._data_stub.CheckAndMutateRow( + request_pb, timeout_seconds) + return resp.predicate_matched def clear_mutations(self): """Removes all currently accumulated mutations on the current row.""" @@ -566,10 +563,9 @@ def commit_modifications(self, timeout_seconds=None): rules=self._rule_pb_list, ) timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.ReadModifyWriteRow.async( - request_pb, timeout_seconds) # We expect a `.data_pb2.Row` - row_response = response.result() + row_response = self.client._data_stub.ReadModifyWriteRow( + request_pb, timeout_seconds) # Reset modifications after commit-ing request. self.clear_modification_rules() diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index 8f6b57b..bbd777a 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -555,7 +555,7 @@ def test__make_data_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, make_beta_stub=mock_make_stub): result = client._make_data_stub() self.assertTrue(result is expected_result) @@ -677,8 +677,7 @@ def _start_method_helper(self, admin): credentials=credentials, admin=admin) stub = _FakeStub() mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, make_stub=mock_make_stub, - make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_beta_stub=mock_make_stub): client.start() self.assertTrue(client._data_stub_internal is stub) From 490849e2dffd77b1d105cfe2db1cb2838bcec3c1 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Nov 2015 15:53:17 -0800 Subject: [PATCH 08/65] Renaming make_beta_stub() as make_stub(). The alias is no longer needed since all stubs have been converted to the new implementation. --- gcloud_bigtable/_helpers.py | 30 +----------------------- gcloud_bigtable/client.py | 18 +++++++-------- gcloud_bigtable/test__helpers.py | 39 -------------------------------- gcloud_bigtable/test_client.py | 10 ++++---- 4 files changed, 15 insertions(+), 82 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index d0e6499..32c5652 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -276,35 +276,7 @@ def get_certs(): def make_stub(client, stub_factory, host, port): """Makes a stub for an RPC service. - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - - :type stub_factory: callable - :param stub_factory: A factory which will create a gRPC stub for - a given service. - - :type host: str - :param host: The host for the service. - - :type port: int - :param port: The port for the service. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: The stub object used to make gRPC requests to the - Data API. - """ - custom_metadata_transformer = MetadataTransformer(client) - return stub_factory(host, port, - metadata_transformer=custom_metadata_transformer, - secure=True, - root_certificates=get_certs()) - - -def make_beta_stub(client, stub_factory, host, port): - """Makes a (beta) stub for an RPC service. - - Uses the beta implementation of gRPC. + Uses / depends on the beta implementation of gRPC. :type client: :class:`.client.Client` :param client: The client that owns the cluster. Provides authorization and diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 672b7a8..482ba32 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -48,7 +48,7 @@ from gcloud_bigtable._generated import bigtable_service_pb2 from gcloud_bigtable._generated import bigtable_table_service_pb2 from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import make_beta_stub +from gcloud_bigtable._helpers import make_stub from gcloud_bigtable.cluster import Cluster @@ -448,8 +448,8 @@ def _make_data_stub(self): :rtype: :class:`grpc.early_adopter.implementations._Stub` :returns: A gRPC stub object. """ - return make_beta_stub(self, DATA_STUB_FACTORY, - DATA_API_HOST, DATA_API_PORT) + return make_stub(self, DATA_STUB_FACTORY, + DATA_API_HOST, DATA_API_PORT) def _make_cluster_stub(self): """Creates gRPC stub to make requests to the Cluster Admin API. @@ -457,8 +457,8 @@ def _make_cluster_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_beta_stub(self, CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return make_stub(self, CLUSTER_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_operations_stub(self): """Creates gRPC stub to make requests to the Operations API. @@ -469,8 +469,8 @@ def _make_operations_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_beta_stub(self, OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return make_stub(self, OPERATIONS_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_table_stub(self): """Creates gRPC stub to make requests to the Table Admin API. @@ -478,8 +478,8 @@ def _make_table_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_beta_stub(self, TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) + return make_stub(self, TABLE_STUB_FACTORY, + TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) def is_started(self): """Check if the client has been started. diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 6039a53..9dd2106 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -411,45 +411,6 @@ def _callFUT(self, credentials, stub_factory, host, port): from gcloud_bigtable._helpers import make_stub return make_stub(credentials, stub_factory, host, port) - def test_it(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - mock_result = object() - custom_factory = _MockCalled(mock_result) - transformed = object() - transformer = _MockCalled(transformed) - - host = 'HOST' - port = 1025 - certs = 'FOOBAR' - client = _MockWithAttachedMethods() - with _Monkey(MUT, get_certs=lambda: certs, - MetadataTransformer=transformer): - result = self._callFUT(client, custom_factory, host, port) - - self.assertTrue(result is mock_result) - custom_factory.check_called( - self, - [(host, port)], - [{ - 'metadata_transformer': transformed, - 'secure': True, - 'root_certificates': certs, - }], - ) - transformer.check_called(self, [(client,)]) - self.assertEqual(client._called, []) - - -class Test_make_beta_stub(unittest2.TestCase): - - def _callFUT(self, credentials, stub_factory, host, port): - from gcloud_bigtable._helpers import make_beta_stub - return make_beta_stub(credentials, stub_factory, host, port) - def test_it(self): from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _MockWithAttachedMethods diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index bbd777a..324914c 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -555,7 +555,7 @@ def test__make_data_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub): result = client._make_data_stub() self.assertTrue(result is expected_result) @@ -583,7 +583,7 @@ def test__make_cluster_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub): result = client._make_cluster_stub() self.assertTrue(result is expected_result) @@ -611,7 +611,7 @@ def test__make_operations_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub): result = client._make_operations_stub() self.assertTrue(result is expected_result) @@ -639,7 +639,7 @@ def test__make_table_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub): result = client._make_table_stub() self.assertTrue(result is expected_result) @@ -677,7 +677,7 @@ def _start_method_helper(self, admin): credentials=credentials, admin=admin) stub = _FakeStub() mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, make_beta_stub=mock_make_stub): + with _Monkey(MUT, make_stub=mock_make_stub): client.start() self.assertTrue(client._data_stub_internal is stub) From 20fb8e5b8c77f6122e5e4bdd4fc9ab4a1c9b2b25 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Nov 2015 08:36:20 -0800 Subject: [PATCH 09/65] Allowing happybase system test to create cluster. If it doesn't exist, the test setup catches an error and then creates the cluster. Also fixing tll->ttl typo. --- system_tests/run_happybase.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py index 082f171..b17fa07 100644 --- a/system_tests/run_happybase.py +++ b/system_tests/run_happybase.py @@ -50,6 +50,7 @@ class Config(object): connection = None table = None + delete_cluster = False def set_hbase_connection(): @@ -58,19 +59,29 @@ def set_hbase_connection(): def set_cloud_bigtable_connection(): + from grpc.framework.interfaces.face.face import NetworkError from gcloud_bigtable import client as client_mod from gcloud_bigtable.happybase import Connection - # NOTE: This assumes that the "gcloud-python" cluster in the - # "us-central1-c" zone already exists for this project. - # We are avoided checking "client.reload()" since the - # alpha version of grcpio does not correctly handle - # request timeouts. client_mod.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' client = client_mod.Client(admin=True) zone = 'us-central1-c' cluster_id = 'gcloud-python' cluster = client.cluster(zone, cluster_id) + client.start() + try: + cluster.reload() + except NetworkError: + cluster.create() + count = 0 + while not cluster.operation_finished(): + count += 1 + if count > 5: + raise NetworkError('Create cluster timed out') + time.sleep(0.5) + # If our test run created a new cluster, then we tell the + # config to clean it up when done. + Config.delete_cluster = True Config.connection = Connection(cluster=cluster) @@ -102,6 +113,8 @@ def tearDownModule(): connection = get_connection() if not USING_HBASE: connection.delete_table(TABLE_NAME) + if Config.delete_cluster: + connection._cluster.delete() connection.close() @@ -740,9 +753,9 @@ def test_put_ttl_eviction(self): else: families = table.families() - cell_tll = TTL_FOR_TEST + cell_ttl = TTL_FOR_TEST chosen_fam = COL_FAM2 - self.assertEqual(families[chosen_fam]['time_to_live'], cell_tll) + self.assertEqual(families[chosen_fam]['time_to_live'], cell_ttl) chosen_col = COL3 self.assertTrue(chosen_col.startswith(chosen_fam + ':')) @@ -756,9 +769,9 @@ def test_put_ttl_eviction(self): self.assertEqual(all_values_before, [value1]) # Make sure we don't sleep for a problematic length. - self.assertTrue(cell_tll < 10) + self.assertTrue(cell_ttl < 10) # Wait for time-to-live eviction to occur. - time.sleep(cell_tll + 0.5) + time.sleep(cell_ttl + 0.5) all_values_after = table.cells(ROW_KEY1, chosen_col) self.assertEqual(all_values_after, []) From d7a6c0676652bbb2e52b4870efc2bc143c967fde Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Nov 2015 08:59:41 -0800 Subject: [PATCH 10/65] Updating CONTRIB/README to drop linuxbrew referencs. --- CONTRIBUTING.md | 17 ++++++----------- README.md | 46 +++++++++++----------------------------------- 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28876e3..1396f1d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,23 +43,18 @@ $ # [sudo] ldconfig ## Testing -Unfortunately, `tox` will fail by default due to the absence of -`grpcio` from `setup.py`. In order to successfully set up a `tox` -test environment (e.g `ENV=py27`), run +`tox` should work without modification, assuming you've installed +on Linux via ```bash -tox -e ${ENV} +$ apt-get install libgrpc-dev ``` -and watch the install fail (missing dependencies), then with the partially -set up environment, execute: +or on Mac OS X via ```bash -BREW_PREFIX=$(brew --prefix) -VERSION=0.10.0a0 -CFLAGS=-I${BREW_PREFIX}/include LDFLAGS=-L${BREW_PREFIX}/lib \ - .tox/${ENV}/bin/pip install grpcio==${VERSION} -unset BREW_PREFIX +$ brew tap grpc/grpc +$ brew install grpc ``` ## False Starts diff --git a/README.md b/README.md index a629913..f8e70a6 100644 --- a/README.md +++ b/README.md @@ -16,54 +16,31 @@ to support via HTTP/1.1. ## Installing gRPC -**NOTE**: These are out-of-date (as of November 18, 2015). - -Make sure you have downloaded [`homebrew`][6] on OS X or -[`linuxbrew`][7] on Linux. (On Linux, also be sure to -add `brew` to `${PATH}` as instructed.) - -First, install the gRPC core (C/C++) library +Before you can install the Python library (`grpcio`) you'll need the +gRPC core library on your system. To install this on Debian Linux: ```bash -curl -fsSL https://goo.gl/getgrpc | bash +$ apt-get install libgrpc-dev ``` -Since this uses `brew` to install, this cannot be run as -root (via `sudo`). - -Next, install the [Python][11] `grpcio` [library][12] via: +or on Mac OS X: ```bash -BREW_PREFIX=$(brew --prefix) -[sudo] CFLAGS=-I${BREW_PREFIX}/include LDFLAGS=-L${BREW_PREFIX}/lib \ -pip install --upgrade grpcio +$ brew tap grpc/grpc +$ brew install grpc ``` -You may wish to run this as root (via `sudo`) so it can be included with -your machine's Python libraries. If not, you'll need to use a Python -[virtual environment][13] so that non-privileged (i.e. non-`sudo`) installs -are allowed. - -Finally, you can install this library via +Once you've installed the system library, install the Python +library via: ```bash -[sudo] pip install -e git+https://github.com/dhermes/gcloud-python-bigtable#egg=gcloud-bigtable +$ pip install --upgrade grpcio ``` -Again, you may wish to install as root or in a virtual environment. - -## Running `gcloud_bigtable` code - -Since the gRPC core is installed via `brew`, the system libraries -are not in a place that Python can readily find them. - -In order to run code that uses `gcloud_bigtable` with these -libraries, you'll need to set the `LD_LIBRARY_PATH` environment -variable: +Finally, you can install this library via ```bash -BREW_PREFIX=$(brew --prefix) -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BREW_PREFIX}/lib +pip install -e git+https://github.com/dhermes/gcloud-python-bigtable#egg=gcloud-bigtable ``` ## Authorization @@ -167,5 +144,4 @@ See [`CONTRIBUTING.md`][3] for instructions on development. [10]: https://github.com/grpc/grpc/issues/2611 [11]: https://github.com/grpc/grpc/tree/master/src/python [12]: https://pypi.python.org/pypi/grpcio -[13]: http://docs.python-guide.org/en/latest/dev/virtualenvs/ [14]: https://console.developers.google.com/ From a92693d440d1a82de349826d184fad478f6b3689 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Nov 2015 12:20:12 -0800 Subject: [PATCH 11/65] Updating MethodMock docstring. --- gcloud_bigtable/_grpc_mocks.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py index 06a6cc9..6c52de4 100644 --- a/gcloud_bigtable/_grpc_mocks.py +++ b/gcloud_bigtable/_grpc_mocks.py @@ -16,13 +16,10 @@ class MethodMock(object): - """Mock for :class:`grpc.framework.alpha._reexport._UnaryUnarySyncAsync`. + """Mock for API method attached to a gRPC stub. - Can also be used for beta equivalent: + In the beta implementation, these are of type. :class:`grpc.framework.crust.implementations._UnaryUnaryMultiCallable` - - May need to be callable and needs to (in our use) have an - ``async`` method. """ def __init__(self, name, factory): From 185ec56ab76aa5495571a2bfe358d28fd074d835 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Nov 2015 12:30:57 -0800 Subject: [PATCH 12/65] Updating test mock to match gcloud-python. --- gcloud_bigtable/_grpc_mocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py index 6c52de4..e66251f 100644 --- a/gcloud_bigtable/_grpc_mocks.py +++ b/gcloud_bigtable/_grpc_mocks.py @@ -29,8 +29,8 @@ def __init__(self, name, factory): def __call__(self, *args, **kwargs): """Sync method meant to mock a gRPC stub request.""" self._factory.method_calls.append((self._name, args, kwargs)) - curr_result = self._factory.results[0] - self._factory.results = self._factory.results[1:] + curr_result, self._factory.results = (self._factory.results[0], + self._factory.results[1:]) return curr_result From d8aeda5b5b53c4adbaa1e13b26b7a11bc7f5185c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 25 Nov 2015 18:05:57 -0800 Subject: [PATCH 13/65] Updating docstring on make_stub. --- gcloud_bigtable/_helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 32c5652..262538d 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -293,8 +293,7 @@ def make_stub(client, stub_factory, host, port): :param port: The port for the service. :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: The stub object used to make gRPC requests to the - Data API. + :returns: The stub object used to make gRPC requests to a given API. """ root_certificates = get_certs() client_credentials = implementations.ssl_client_credentials( From 0f26ad2b3a6eb7f9e988d7bb5676e589eb7c6a49 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 27 Nov 2015 10:01:38 -0800 Subject: [PATCH 14/65] Removing host from beta stub factory. It is only needed as an optional over-ride, otherwise the host from the channel is sufficient. --- gcloud_bigtable/_helpers.py | 2 +- gcloud_bigtable/test__helpers.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 262538d..7c08179 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -301,7 +301,7 @@ def make_stub(client, stub_factory, host, port): channel = implementations.secure_channel( host, port, client_credentials) custom_metadata_transformer = MetadataTransformer(client) - return stub_factory(channel, host=host, + return stub_factory(channel, metadata_transformer=custom_metadata_transformer) diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 9dd2106..43add36 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -440,7 +440,6 @@ def test_it(self): self, [(channel,)], [{ - 'host': host, 'metadata_transformer': transformed, }], ) From 647bcc34ce38e6f1ac42de8b77184755cbace50f Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 30 Nov 2015 15:14:14 -0800 Subject: [PATCH 15/65] Upgrading for Pylint and syncing with gcloud-python. --- gcloud_bigtable/_helpers.py | 35 --------------------- gcloud_bigtable/client.py | 6 ++-- gcloud_bigtable/cluster.py | 31 +++++++++++++++--- gcloud_bigtable/happybase/connection.py | 2 ++ gcloud_bigtable/happybase/pool.py | 3 +- gcloud_bigtable/happybase/table.py | 3 +- gcloud_bigtable/row.py | 3 +- gcloud_bigtable/test__helpers.py | 42 ------------------------- gcloud_bigtable/test_client.py | 1 + gcloud_bigtable/test_cluster.py | 22 +++++++++++++ pylintrc_default | 8 ++++- run_pylint.py | 1 + system_tests/run.py | 21 +++++++------ system_tests/run_happybase.py | 2 ++ tox.ini | 2 +- 15 files changed, 83 insertions(+), 99 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 7c08179..7e4cd66 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -95,41 +95,6 @@ def _pb_timestamp_to_datetime(timestamp): ) -def _require_pb_property(message_pb, property_name, value): - """Check that a property agrees with the value on the message. - - :type message_pb: :class:`google.protobuf.message.Message` - :param message_pb: The message to check for ``property_name``. - - :type property_name: str - :param property_name: The property value to check against. - - :type value: object or :data:`NoneType ` - :param value: The value to check against the cluster. If :data:`None`, - will not be checked. - - :rtype: object - :returns: The value of ``property_name`` set on ``message_pb``. - :raises: :class:`ValueError ` if the result returned - from the ``message_pb`` does not contain the ``property_name`` - value or if the value returned disagrees with the ``value`` - passed with the request (if that value is not null). - """ - # Make sure `property_name` is set on the response. - # NOTE: HasField() doesn't work in protobuf>=3.0.0a3 - all_fields = set([field.name for field in message_pb._fields]) - if property_name not in all_fields: - raise ValueError('Message does not contain %s.' % (property_name,)) - property_val = getattr(message_pb, property_name) - if value is None: - value = property_val - elif value != property_val: - raise ValueError('Message returned %s value disagreeing ' - 'with value passed in.' % (property_name,)) - - return value - - def _parse_pb_any_to_native(any_val, expected_type=None): """Convert a serialized "google.protobuf.Any" value to actual type. diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 482ba32..653dba8 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -29,9 +29,10 @@ import copy import os -import six import socket +import six + from oauth2client.client import GoogleCredentials from oauth2client.client import SignedJwtAssertionCredentials from oauth2client.client import _get_application_default_credential_from_file @@ -330,7 +331,8 @@ def from_service_account_p12(cls, client_email, private_key_path, """ credentials = SignedJwtAssertionCredentials( service_account_name=client_email, - private_key=_get_contents(private_key_path)) + private_key=_get_contents(private_key_path), + scope=None) return cls(credentials=credentials, project=project, read_only=read_only, admin=admin) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 9a15611..a4b682d 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -25,7 +25,6 @@ from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._helpers import _parse_pb_any_to_native from gcloud_bigtable._helpers import _pb_timestamp_to_datetime -from gcloud_bigtable._helpers import _require_pb_property from gcloud_bigtable.table import Table @@ -37,6 +36,30 @@ r'(?P\d+)$') +def _get_pb_property_value(message_pb, property_name): + """Return a message field value. + + :type message_pb: :class:`google.protobuf.message.Message` + :param message_pb: The message to check for ``property_name``. + + :type property_name: str + :param property_name: The property value to check against. + + :rtype: object + :returns: The value of ``property_name`` set on ``message_pb``. + :raises: :class:`ValueError ` if the result returned + from the ``message_pb`` does not contain the ``property_name`` + value. + """ + # Make sure `property_name` is set on the response. + # NOTE: As of proto3, HasField() only works for message fields, not for + # singular (non-message) fields. + all_fields = set([field.name for field in message_pb._fields]) + if property_name not in all_fields: + raise ValueError('Message does not contain %s.' % (property_name,)) + return getattr(message_pb, property_name) + + def _prepare_create_request(cluster): """Creates a protobuf request for a CreateCluster request. @@ -136,10 +159,8 @@ def __init__(self, zone, cluster_id, client, self._operation_begin = None def _update_from_pb(self, cluster_pb): - self.display_name = _require_pb_property( - cluster_pb, 'display_name', None) - self.serve_nodes = _require_pb_property( - cluster_pb, 'serve_nodes', None) + self.display_name = _get_pb_property_value(cluster_pb, 'display_name') + self.serve_nodes = _get_pb_property_value(cluster_pb, 'serve_nodes') @classmethod def from_pb(cls, cluster_pb, client): diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 7876f05..95dd525 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -102,10 +102,12 @@ def _parse_family_option(option): """ result = option if isinstance(result, dict): + # pylint: disable=unneeded-not if not set(result.keys()) <= set(['max_versions', 'time_to_live']): raise ValueError('Cloud Bigtable only supports max_versions and ' 'time_to_live column family settings', 'Received', result.keys()) + # pylint: enable=unneeded-not max_num_versions = result.get('max_versions') max_age = None diff --git a/gcloud_bigtable/happybase/pool.py b/gcloud_bigtable/happybase/pool.py index 330c5de..e77d7ba 100644 --- a/gcloud_bigtable/happybase/pool.py +++ b/gcloud_bigtable/happybase/pool.py @@ -16,9 +16,10 @@ import contextlib -import six import threading +import six + from gcloud_bigtable.happybase.connection import Connection from gcloud_bigtable.happybase.connection import _get_cluster diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index c496740..c7daf1b 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -15,9 +15,10 @@ """Google Cloud Bigtable HappyBase table module.""" -import six import struct +import six + from gcloud_bigtable._helpers import _microseconds_to_timestamp from gcloud_bigtable._helpers import _timestamp_to_microseconds from gcloud_bigtable._helpers import _to_bytes diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 35ed9c4..c070dfc 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -15,9 +15,10 @@ """User friendly container for Google Cloud Bigtable Row.""" -import six import struct +import six + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 43add36..06107b3 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -94,48 +94,6 @@ def test_it(self): self.assertEqual(self._callFUT(timestamp), dt_stamp) -class Test__require_pb_property(unittest2.TestCase): - - def _callFUT(self, message_pb, property_name, value): - from gcloud_bigtable._helpers import _require_pb_property - return _require_pb_property(message_pb, property_name, value) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - cluster_pb = data_pb2.Cluster(serve_nodes=serve_nodes) - result = self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - self.assertEqual(result, serve_nodes) - - def test_with_null_value_passed_in(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = None - actual_serve_nodes = 119 - cluster_pb = data_pb2.Cluster(serve_nodes=actual_serve_nodes) - result = self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - self.assertEqual(result, actual_serve_nodes) - - def test_with_value_unset_on_pb(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - cluster_pb = data_pb2.Cluster() - with self.assertRaises(ValueError): - self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - - def test_with_values_disagreeing(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - other_serve_nodes = 1000 - self.assertNotEqual(serve_nodes, other_serve_nodes) - cluster_pb = data_pb2.Cluster(serve_nodes=other_serve_nodes) - with self.assertRaises(ValueError): - self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - - class Test__parse_pb_any_to_native(unittest2.TestCase): def _callFUT(self, any_val, expected_type=None): diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index 324914c..a2eb0af 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -368,6 +368,7 @@ def test_from_service_account_p12(self): signed_creds_kw = { 'private_key': private_key, 'service_account_name': client_email, + 'scope': None, } signed_creds.check_called(self, [()], [signed_creds_kw]) # Load private key (via _get_contents) from the key path. diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index cbdf373..ea4f885 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -21,6 +21,28 @@ CLUSTER_ID = 'cluster-id' +class Test__get_pb_property_value(unittest2.TestCase): + + def _callFUT(self, message_pb, property_name): + from gcloud_bigtable.cluster import _get_pb_property_value + return _get_pb_property_value(message_pb, property_name) + + def test_it(self): + from gcloud_bigtable._generated import ( + bigtable_cluster_data_pb2 as data_pb2) + serve_nodes = 119 + cluster_pb = data_pb2.Cluster(serve_nodes=serve_nodes) + result = self._callFUT(cluster_pb, 'serve_nodes') + self.assertEqual(result, serve_nodes) + + def test_with_value_unset_on_pb(self): + from gcloud_bigtable._generated import ( + bigtable_cluster_data_pb2 as data_pb2) + cluster_pb = data_pb2.Cluster() + with self.assertRaises(ValueError): + self._callFUT(cluster_pb, 'serve_nodes') + + class Test__prepare_create_request(unittest2.TestCase): def _callFUT(self, cluster): diff --git a/pylintrc_default b/pylintrc_default index 1be8524..6fa9050 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -96,6 +96,11 @@ load-plugins=pylint.extensions.check_docs # @_lazy_property_deco # def dataset_id(): # ... +# - redefined-variable-type: This error is overzealous and complains at e.g. +# if some_prop: +# return int(value) +# else: +# return float(value) disable = maybe-no-member, no-member, @@ -104,6 +109,7 @@ disable = similarities, star-args, method-hidden, + redefined-variable-type, [REPORTS] @@ -199,7 +205,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1500 +max-module-lines=1540 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). diff --git a/run_pylint.py b/run_pylint.py index 99b0647..5dbea3a 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -50,6 +50,7 @@ 'too-many-locals', 'too-many-public-methods', 'unbalanced-tuple-unpacking', + 'abstract-method', ] TEST_RC_ADDITIONS = { 'MESSAGES CONTROL': { diff --git a/system_tests/run.py b/system_tests/run.py index 2755f37..97bad29 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -17,8 +17,9 @@ import datetime import operator import os -import pytz import time + +import pytz import unittest2 from oauth2client.client import GoogleCredentials @@ -165,13 +166,13 @@ def test_update(self): class TestTableAdminAPI(unittest2.TestCase): @classmethod - def setUpClass(self): - self._table = CLUSTER.table(TABLE_ID) - self._table.create() + def setUpClass(cls): + cls._table = CLUSTER.table(TABLE_ID) + cls._table.create() @classmethod - def tearDownClass(self): - self._table.delete() + def tearDownClass(cls): + cls._table.delete() def setUp(self): self.tables_to_delete = [] @@ -245,16 +246,16 @@ def test_delete_column_family(self): class TestDataAPI(unittest2.TestCase): @classmethod - def setUpClass(self): - self._table = table = CLUSTER.table(TABLE_ID) + def setUpClass(cls): + cls._table = table = CLUSTER.table(TABLE_ID) table.create() table.column_family(COLUMN_FAMILY_ID1).create() table.column_family(COLUMN_FAMILY_ID2).create() @classmethod - def tearDownClass(self): + def tearDownClass(cls): # Will also delete any data contained in the table. - self._table.delete() + cls._table.delete() def setUp(self): self.rows_to_delete = [] diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py index b17fa07..803963a 100644 --- a/system_tests/run_happybase.py +++ b/system_tests/run_happybase.py @@ -59,9 +59,11 @@ def set_hbase_connection(): def set_cloud_bigtable_connection(): + # pylint: disable=no-name-in-module from grpc.framework.interfaces.face.face import NetworkError from gcloud_bigtable import client as client_mod from gcloud_bigtable.happybase import Connection + # pylint: enable=no-name-in-module client_mod.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' client = client_mod.Client(admin=True) diff --git a/tox.ini b/tox.ini index ee054c5..c83cd1b 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ commands = python run_pylint.py deps = pep8 - -ehg+https://bitbucket.org/logilab/pylint@33e334be064c#egg=pylint + pylint unittest2 passenv = GCLOUD_* From d864c3ef4c903a869f8d07879eb290c2c693dfb5 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 1 Dec 2015 09:47:39 -0800 Subject: [PATCH 16/65] Moving some code around to stay up-to-date with gcloud-python. --- gcloud_bigtable/__init__.py | 2 ++ gcloud_bigtable/client.py | 2 +- gcloud_bigtable/cluster.py | 31 ++++++++++++++++--------------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/gcloud_bigtable/__init__.py b/gcloud_bigtable/__init__.py index 251590d..8393025 100644 --- a/gcloud_bigtable/__init__.py +++ b/gcloud_bigtable/__init__.py @@ -24,3 +24,5 @@ 'Python locate the libraries.') else: raise + +from gcloud_bigtable.client import Client diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 653dba8..4b98107 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -200,7 +200,7 @@ class Client(object): :class:`OAuth2Credentials ` or :data:`NoneType ` :param credentials: (Optional) The OAuth2 Credentials to use for this - cluster. If not provided, defaulst to the Google + cluster. If not provided, defaults to the Google Application Default Credentials. :type read_only: bool diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index a4b682d..8ce4e17 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -158,6 +158,17 @@ def __init__(self, zone, cluster_id, client, self._operation_id = None self._operation_begin = None + def table(self, table_id): + """Factory to create a table associated with this cluster. + + :type table_id: str + :param table_id: The ID of the table. + + :rtype: :class:`Table ` + :returns: The table owned by this cluster. + """ + return Table(table_id, self) + def _update_from_pb(self, cluster_pb): self.display_name = _get_pb_property_value(cluster_pb, 'display_name') self.serve_nodes = _get_pb_property_value(cluster_pb, 'serve_nodes') @@ -250,25 +261,15 @@ def name(self): return (self.client.project_name + '/zones/' + self.zone + '/clusters/' + self.cluster_id) - def table(self, table_id): - """Factory to create a table associated with this cluster. - - :type table_id: str - :param table_id: The ID of the table. - - :rtype: :class:`Table ` - :returns: The table owned by this cluster. - """ - return Table(table_id, self) - def __eq__(self, other): if not isinstance(other, self.__class__): return False # NOTE: This does not compare the configuration values, such as - # the serve_nodes or display_name. This is intentional, since - # the same cluster can be in different states if not - # synchronized. This suggests we should use `project` - # instead of `client` for the third comparison. + # the serve_nodes or display_name. Instead, it only compares + # identifying values zone, cluster ID and client. This is + # intentional, since the same cluster can be in different states + # if not synchronized. Clusters with similar zone/cluster + # settings but different clients can't be used in the same way. return (other.zone == self.zone and other.cluster_id == self.cluster_id and other.client == self.client) From d16fa35c4070445d38a5103914c1c207487eb7ed Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 1 Dec 2015 10:43:34 -0800 Subject: [PATCH 17/65] Adding support for cert-file from OS X. --- gcloud_bigtable/_helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 7e4cd66..027e01a 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -20,6 +20,8 @@ import datetime +import platform + import pytz import six @@ -47,7 +49,12 @@ EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) # See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 # for appropriate values on other systems. -SSL_CERT_FILE = '/etc/ssl/certs/ca-certificates.crt' +_PLAT_SYS = platform.system() +SSL_CERT_FILE = None +if _PLAT_SYS == 'Linux': + SSL_CERT_FILE = '/usr/local/etc/openssl/cert.pem' +elif _PLAT_SYS == 'Darwin': # pragma: NO COVER + SSL_CERT_FILE = '/etc/ssl/certs/ca-certificates.crt' class MetadataTransformer(object): From c95d0cfacbe7e64e760363f3d5498b4c31e7a270 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 2 Dec 2015 06:56:39 -0800 Subject: [PATCH 18/65] Swapping (incorrect) cert files for OS X/Linux. Also updating a test-case to stay current with upstream in gcloud-python. --- gcloud_bigtable/_helpers.py | 4 ++-- gcloud_bigtable/test_cluster.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 027e01a..b64376b 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -52,9 +52,9 @@ _PLAT_SYS = platform.system() SSL_CERT_FILE = None if _PLAT_SYS == 'Linux': - SSL_CERT_FILE = '/usr/local/etc/openssl/cert.pem' -elif _PLAT_SYS == 'Darwin': # pragma: NO COVER SSL_CERT_FILE = '/etc/ssl/certs/ca-certificates.crt' +elif _PLAT_SYS == 'Darwin': # pragma: NO COVER + SSL_CERT_FILE = '/usr/local/etc/openssl/cert.pem' class MetadataTransformer(object): diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index ea4f885..9f4fdce 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -55,7 +55,8 @@ def test_it(self): from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable.cluster import Cluster - display_name = 'DISPLAY_NAME' + + display_name = u'DISPLAY_NAME' serve_nodes = 8 cluster = Cluster(ZONE, CLUSTER_ID, _Client(PROJECT), From 698b257758b6c171634580640bc52f663ffbed36 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 4 Dec 2015 16:06:47 -0800 Subject: [PATCH 19/65] Updating with upstream. --- gcloud_bigtable/_helpers.py | 2 +- gcloud_bigtable/cluster.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index b64376b..0512357 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -87,7 +87,7 @@ class AuthInfo(object): def _pb_timestamp_to_datetime(timestamp): """Convert a Timestamp protobuf to a datetime object. - :type timestamp: :class:`._generated.timestamp_pb.Timestamp` + :type timestamp: :class:`._generated.timestamp_pb2.Timestamp` :param timestamp: A Google returned timestamp protobuf. :rtype: :class:`datetime.datetime` diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 8ce4e17..6082e06 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -396,6 +396,28 @@ def update(self, timeout_seconds=None): def delete(self, timeout_seconds=None): """Delete this cluster. + Marks a cluster and all of its tables for permanent deletion in 7 days. + + Immediately upon completion of the request: + + * Billing will cease for all of the cluster's reserved resources. + * The cluster's ``delete_time`` field will be set 7 days in the future. + + Soon afterward: + + * All tables within the cluster will become unavailable. + + Prior to the cluster's ``delete_time``: + + * The cluster can be recovered with a call to ``UndeleteCluster``. + * All other attempts to modify or delete the cluster will be rejected. + + At the cluster's ``delete_time``: + + * The cluster and **all of its tables** will immediately and + irrevocably disappear from the API, and their data will be + permanently deleted. + :type timeout_seconds: int :param timeout_seconds: Number of seconds for request time-out. If not passed, defaults to value set on From fa466c5cbf7d2477d149c6dfdc4cf617bd5f75e0 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 7 Dec 2015 18:28:01 -0800 Subject: [PATCH 20/65] Moving over Operation class from upstream. Currently breaks the way operation_finished() is handled. Will be fixed upstream and ported here in short order. --- gcloud_bigtable/cluster.py | 41 +++++++++++++++++++++-- gcloud_bigtable/test_cluster.py | 59 +++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 6082e06..08eea89 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -108,6 +108,38 @@ def _process_operation(operation_pb): return operation_id, operation_begin +class Operation(object): + """Representation of a Google API Long-Running Operation. + + In particular, the + + :type op_type: str + :param op_type: The type of operation being performed. Expect + ``create``, ``update`` or ``undelete``. + + :type op_id: int + :param op_id: The ID of the operation. + + :type begin: :class:`datetime.datetime` + :param begin: The time when the operation was started. + """ + + def __init__(self, op_type, op_id, begin): + self.op_type = op_type + self.op_id = op_id + self.begin = begin + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return (other.op_type == self.op_type and + other.op_id == self.op_id and + other.begin == self.begin) + + def __ne__(self, other): + return not self.__eq__(other) + + class Cluster(object): """Representation of a Google Cloud Bigtable Cluster. @@ -348,6 +380,10 @@ def create(self, timeout_seconds=None): :param timeout_seconds: Number of seconds for request time-out. If not passed, defaults to value set on cluster. + + :rtype: :class:`Operation` + :returns: The long-running operation corresponding to the + create operation. """ request_pb = _prepare_create_request(self) timeout_seconds = timeout_seconds or self.timeout_seconds @@ -355,9 +391,8 @@ def create(self, timeout_seconds=None): cluster_pb = self.client._cluster_stub.CreateCluster( request_pb, timeout_seconds) - self._operation_type = 'create' - self._operation_id, self._operation_begin = _process_operation( - cluster_pb.current_operation) + op_id, op_begin = _process_operation(cluster_pb.current_operation) + return Operation('create', op_id, op_begin) def update(self, timeout_seconds=None): """Update this cluster. diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index 9f4fdce..b68a578 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -118,6 +118,56 @@ def test_failure(self): self._callFUT(cluster) +class TestOperation(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.cluster import Operation + return Operation + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + import datetime + op_type = 'fake-op' + op_id = 8915 + begin = datetime.datetime(2015, 10, 22, 1, 1) + operation = self._makeOne(op_type, op_id, begin) + + self.assertEqual(operation.op_type, op_type) + self.assertEqual(operation.op_id, op_id) + self.assertEqual(operation.begin, begin) + + def test___eq__(self): + import datetime + op_type = 'fake-op' + op_id = 8915 + begin = datetime.datetime(2015, 10, 22, 1, 1) + operation1 = self._makeOne(op_type, op_id, begin) + operation2 = self._makeOne(op_type, op_id, begin) + self.assertEqual(operation1, operation2) + + def test___eq__type_differ(self): + operation1 = self._makeOne('foo', 123, None) + operation2 = object() + self.assertNotEqual(operation1, operation2) + + def test___ne__same_value(self): + import datetime + op_type = 'fake-op' + op_id = 8915 + begin = datetime.datetime(2015, 10, 22, 1, 1) + operation1 = self._makeOne(op_type, op_id, begin) + operation2 = self._makeOne(op_type, op_id, begin) + comparison_val = (operation1 != operation2) + self.assertFalse(comparison_val) + + def test___ne__(self): + operation1 = self._makeOne('foo', 123, None) + operation2 = self._makeOne('bar', 456, None) + self.assertNotEqual(operation1, operation2) + + class TestCluster(unittest2.TestCase): def _getTargetClass(self): @@ -420,13 +470,13 @@ def test_create(self): client._cluster_stub = stub = StubMock(response_pb) # Create expected_result. - expected_result = None + op_id = 5678 + op_begin = object() + expected_result = MUT.Operation('create', op_id, op_begin) # Perform the method and check the result. timeout_seconds = 578 mock_prepare_create_request = _MockCalled(request_pb) - op_id = 5678 - op_begin = object() mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _prepare_create_request=mock_prepare_create_request, _process_operation=mock_process_operation): @@ -438,9 +488,6 @@ def test_create(self): (request_pb, timeout_seconds), {}, )]) - self.assertEqual(cluster._operation_type, 'create') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) mock_prepare_create_request.check_called(self, [(cluster,)]) mock_process_operation.check_called(self, [(current_op,)]) From e6a49fc20b386d5ae2506e8b16f5bcb916fd210e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 8 Dec 2015 22:18:47 -0800 Subject: [PATCH 21/65] Brining Operation changes upstream from gcloud-python. --- gcloud_bigtable/cluster.py | 129 +++++++++++++++----------- gcloud_bigtable/test_cluster.py | 159 +++++++++++++++++--------------- 2 files changed, 160 insertions(+), 128 deletions(-) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 08eea89..869d3d9 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -111,7 +111,8 @@ def _process_operation(operation_pb): class Operation(object): """Representation of a Google API Long-Running Operation. - In particular, the + In particular, these will be the result of operations on + clusters using the Cloud Bigtable API. :type op_type: str :param op_type: The type of operation being performed. Expect @@ -122,23 +123,60 @@ class Operation(object): :type begin: :class:`datetime.datetime` :param begin: The time when the operation was started. + + :type cluster: :class:`Cluster` + :param cluster: The cluster that created the operation. """ - def __init__(self, op_type, op_id, begin): + def __init__(self, op_type, op_id, begin, cluster=None): self.op_type = op_type self.op_id = op_id self.begin = begin + self._cluster = cluster + self._complete = False def __eq__(self, other): if not isinstance(other, self.__class__): return False return (other.op_type == self.op_type and other.op_id == self.op_id and - other.begin == self.begin) + other.begin == self.begin and + other._cluster == self._cluster and + other._complete == self._complete) def __ne__(self, other): return not self.__eq__(other) + def finished(self, timeout_seconds=None): + """Check if the operation has finished. + + :type timeout_seconds: int + :param timeout_seconds: Number of seconds for request time-out. + If not passed, defaults to value set on + cluster. + + :rtype: bool + :returns: A boolean indicating if the current operation has completed. + :raises: :class:`ValueError ` if the operation + has already completed. + """ + if self._complete: + raise ValueError('The operation has completed.') + + operation_name = ('operations/' + self._cluster.name + + '/operations/%d' % (self.op_id,)) + request_pb = operations_pb2.GetOperationRequest(name=operation_name) + timeout_seconds = timeout_seconds or self._cluster.timeout_seconds + # We expact a `._generated.operations_pb2.Operation`. + operation_pb = self._cluster._client._operations_stub.GetOperation( + request_pb, timeout_seconds) + + if operation_pb.done: + self._complete = True + return True + else: + return False + class Cluster(object): """Representation of a Google Cloud Bigtable Cluster. @@ -147,8 +185,6 @@ class Cluster(object): * :meth:`reload` itself * :meth:`create` itself - * Check if an :meth:`operation_finished` (each of :meth:`create`, - :meth:`update` and :meth:`undelete` return with long-running operations) * :meth:`update` itself * :meth:`delete` itself * :meth:`undelete` itself @@ -186,9 +222,6 @@ def __init__(self, zone, cluster_id, client, self.display_name = display_name or cluster_id self.serve_nodes = serve_nodes self._client = client - self._operation_type = None - self._operation_id = None - self._operation_begin = None def table(self, table_id): """Factory to create a table associated with this cluster. @@ -244,9 +277,9 @@ def copy(self): :returns: A copy of the current cluster. """ new_client = self.client.copy() - return Cluster(self.zone, self.cluster_id, new_client, - display_name=self.display_name, - serve_nodes=self.serve_nodes) + return self.__class__(self.zone, self.cluster_id, new_client, + display_name=self.display_name, + serve_nodes=self.serve_nodes) @property def client(self): @@ -327,38 +360,6 @@ def reload(self, timeout_seconds=None): # cluster ID on the response match the request. self._update_from_pb(cluster_pb) - def operation_finished(self, timeout_seconds=None): - """Check if the current operation has finished. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - - :rtype: bool - :returns: A boolean indicating if the current operation has completed. - :raises: :class:`ValueError ` if there is no - current operation set. - """ - if self._operation_id is None: - raise ValueError('There is no current operation.') - - operation_name = ('operations/' + self.name + - '/operations/%d' % (self._operation_id,)) - request_pb = operations_pb2.GetOperationRequest(name=operation_name) - timeout_seconds = timeout_seconds or self.timeout_seconds - # We expact a `._generated.operations_pb2.Operation`. - operation_pb = self.client._operations_stub.GetOperation( - request_pb, timeout_seconds) - - if operation_pb.done: - self._operation_type = None - self._operation_id = None - self._operation_begin = None - return True - else: - return False - def create(self, timeout_seconds=None): """Create this cluster. @@ -413,6 +414,10 @@ def update(self, timeout_seconds=None): :param timeout_seconds: Number of seconds for request time-out. If not passed, defaults to value set on cluster. + + :rtype: :class:`Operation` + :returns: The long-running operation corresponding to the + update operation. """ request_pb = data_pb2.Cluster( name=self.name, @@ -424,9 +429,8 @@ def update(self, timeout_seconds=None): cluster_pb = self.client._cluster_stub.UpdateCluster( request_pb, timeout_seconds) - self._operation_type = 'update' - self._operation_id, self._operation_begin = _process_operation( - cluster_pb.current_operation) + op_id, op_begin = _process_operation(cluster_pb.current_operation) + return Operation('update', op_id, op_begin) def delete(self, timeout_seconds=None): """Delete this cluster. @@ -466,10 +470,32 @@ def delete(self, timeout_seconds=None): def undelete(self, timeout_seconds=None): """Undelete this cluster. + Cancels the scheduled deletion of an cluster and begins preparing it to + resume serving. The returned operation will also be embedded as the + cluster's ``current_operation``. + + Immediately upon completion of this request: + + * The cluster's ``delete_time`` field will be unset, protecting it from + automatic deletion. + + Until completion of the returned operation: + + * The operation cannot be cancelled. + + Upon completion of the returned operation: + + * Billing for the cluster's resources will resume. + * All tables within the cluster will be available. + :type timeout_seconds: int :param timeout_seconds: Number of seconds for request time-out. If not passed, defaults to value set on cluster. + + :rtype: :class:`Operation` + :returns: The long-running operation corresponding to the + undelete operation. """ request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) timeout_seconds = timeout_seconds or self.timeout_seconds @@ -477,9 +503,8 @@ def undelete(self, timeout_seconds=None): operation_pb2 = self.client._cluster_stub.UndeleteCluster( request_pb, timeout_seconds) - self._operation_type = 'undelete' - self._operation_id, self._operation_begin = _process_operation( - operation_pb2) + op_id, op_begin = _process_operation(operation_pb2) + return Operation('undelete', op_id, op_begin) def list_tables(self, timeout_seconds=None): """List the tables in this cluster. @@ -502,11 +527,11 @@ def list_tables(self, timeout_seconds=None): result = [] for table_pb in table_list_pb.tables: - before, table_id = table_pb.name.split( - self.name + '/tables/', 1) - if before != '': + table_prefix = self.name + '/tables/' + if not table_pb.name.startswith(table_prefix): raise ValueError('Table name %s not of expected format' % ( table_pb.name,)) + table_id = table_pb.name[len(table_prefix):] result.append(self.table(table_id)) return result diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index b68a578..7792022 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -127,24 +127,34 @@ def _getTargetClass(self): def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) - def test_constructor(self): + def _constructor_test_helper(self, cluster=None): import datetime op_type = 'fake-op' op_id = 8915 begin = datetime.datetime(2015, 10, 22, 1, 1) - operation = self._makeOne(op_type, op_id, begin) + operation = self._makeOne(op_type, op_id, begin, cluster=cluster) self.assertEqual(operation.op_type, op_type) self.assertEqual(operation.op_id, op_id) self.assertEqual(operation.begin, begin) + self.assertEqual(operation._cluster, cluster) + self.assertFalse(operation._complete) + + def test_constructor_defaults(self): + self._constructor_test_helper() + + def test_constructor_explicit_cluster(self): + cluster = object() + self._constructor_test_helper(cluster=cluster) def test___eq__(self): import datetime op_type = 'fake-op' op_id = 8915 begin = datetime.datetime(2015, 10, 22, 1, 1) - operation1 = self._makeOne(op_type, op_id, begin) - operation2 = self._makeOne(op_type, op_id, begin) + cluster = object() + operation1 = self._makeOne(op_type, op_id, begin, cluster=cluster) + operation2 = self._makeOne(op_type, op_id, begin, cluster=cluster) self.assertEqual(operation1, operation2) def test___eq__type_differ(self): @@ -157,8 +167,9 @@ def test___ne__same_value(self): op_type = 'fake-op' op_id = 8915 begin = datetime.datetime(2015, 10, 22, 1, 1) - operation1 = self._makeOne(op_type, op_id, begin) - operation2 = self._makeOne(op_type, op_id, begin) + cluster = object() + operation1 = self._makeOne(op_type, op_id, begin, cluster=cluster) + operation2 = self._makeOne(op_type, op_id, begin, cluster=cluster) comparison_val = (operation1 != operation2) self.assertFalse(comparison_val) @@ -167,6 +178,66 @@ def test___ne__(self): operation2 = self._makeOne('bar', 456, None) self.assertNotEqual(operation1, operation2) + def test_finished_without_operation(self): + operation = self._makeOne(None, None, None) + operation._complete = True + with self.assertRaises(ValueError): + operation.finished() + + def _finished_helper(self, done): + import datetime + from gcloud_bigtable._generated import operations_pb2 + from gcloud_bigtable._grpc_mocks import StubMock + from gcloud_bigtable.cluster import Cluster + + project = 'PROJECT' + zone = 'zone' + cluster_id = 'cluster-id' + op_type = 'fake-op' + op_id = 789 + begin = datetime.datetime(2015, 10, 22, 1, 1) + timeout_seconds = 1 + + client = _Client(project) + cluster = Cluster(zone, cluster_id, client) + operation = self._makeOne(op_type, op_id, begin, cluster=cluster) + + # Create request_pb + op_name = ('operations/projects/' + project + '/zones/' + + zone + '/clusters/' + cluster_id + + '/operations/%d' % (op_id,)) + request_pb = operations_pb2.GetOperationRequest(name=op_name) + + # Create response_pb + response_pb = operations_pb2.Operation(done=done) + + # Patch the stub used by the API method. + client._operations_stub = stub = StubMock(response_pb) + + # Create expected_result. + expected_result = done + + # Perform the method and check the result. + result = operation.finished(timeout_seconds=timeout_seconds) + + self.assertEqual(result, expected_result) + self.assertEqual(stub.method_calls, [( + 'GetOperation', + (request_pb, timeout_seconds), + {}, + )]) + + if done: + self.assertTrue(operation._complete) + else: + self.assertFalse(operation._complete) + + def test_finished(self): + self._finished_helper(done=True) + + def test_finished_not_done(self): + self._finished_helper(done=False) + class TestCluster(unittest2.TestCase): @@ -388,64 +459,6 @@ def test_reload(self): {}, )]) - def test_operation_finished_without_operation(self): - cluster = self._makeOne(ZONE, CLUSTER_ID, None) - self.assertEqual(cluster._operation_type, None) - with self.assertRaises(ValueError): - cluster.operation_finished() - - def _operation_finished_helper(self, done): - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Patch up the cluster's operation attributes. - cluster._operation_id = op_id = 789 - cluster._operation_begin = op_begin = object() - cluster._operation_type = op_type = object() - - # Create request_pb - op_name = ('operations/projects/' + PROJECT + '/zones/' + - ZONE + '/clusters/' + CLUSTER_ID + - '/operations/%d' % (op_id,)) - request_pb = operations_pb2.GetOperationRequest(name=op_name) - - # Create response_pb - response_pb = operations_pb2.Operation(done=done) - - # Patch the stub used by the API method. - client._operations_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = done - - # Perform the method and check the result. - timeout_seconds = 1 - result = cluster.operation_finished(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetOperation', - (request_pb, timeout_seconds), - {}, - )]) - - if done: - self.assertEqual(cluster._operation_type, None) - self.assertEqual(cluster._operation_id, None) - self.assertEqual(cluster._operation_begin, None) - else: - self.assertEqual(cluster._operation_type, op_type) - self.assertEqual(cluster._operation_id, op_id) - self.assertEqual(cluster._operation_begin, op_begin) - - def test_operation_finished(self): - self._operation_finished_helper(done=True) - - def test_operation_finished_not_done(self): - self._operation_finished_helper(done=False) - def test_create(self): from gcloud_bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) @@ -524,12 +537,12 @@ def test_update(self): client._cluster_stub = stub = StubMock(response_pb) # Create expected_result. - expected_result = None + op_id = 5678 + op_begin = object() + expected_result = MUT.Operation('update', op_id, op_begin) # We must create the cluster object with the client passed in. timeout_seconds = 9 - op_id = 5678 - op_begin = object() mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _process_operation=mock_process_operation): @@ -541,9 +554,6 @@ def test_update(self): (request_pb, timeout_seconds), {}, )]) - self.assertEqual(cluster._operation_type, 'update') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) mock_process_operation.check_called(self, [(current_op,)]) def test_delete(self): @@ -603,12 +613,12 @@ def test_undelete(self): client._cluster_stub = stub = StubMock(response_pb) # Create expected_result. - expected_result = None + op_id = 5678 + op_begin = object() + expected_result = MUT.Operation('undelete', op_id, op_begin) # Perform the method and check the result. timeout_seconds = 78 - op_id = 5678 - op_begin = object() mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _process_operation=mock_process_operation): @@ -620,9 +630,6 @@ def test_undelete(self): (request_pb, timeout_seconds), {}, )]) - self.assertEqual(cluster._operation_type, 'undelete') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) mock_process_operation.check_called(self, [(response_pb,)]) def _list_tables_helper(self, table_id, table_name=None): From 4ba9ae24fd97514e7dbc2d61c51a1d4d353eddf4 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 8 Dec 2015 22:28:17 -0800 Subject: [PATCH 22/65] Updating system tests for Operation changes. Also passing the current cluster when creating new operations. --- gcloud_bigtable/cluster.py | 6 +++--- system_tests/run.py | 12 ++++++------ system_tests/run_happybase.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 869d3d9..f937258 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -393,7 +393,7 @@ def create(self, timeout_seconds=None): request_pb, timeout_seconds) op_id, op_begin = _process_operation(cluster_pb.current_operation) - return Operation('create', op_id, op_begin) + return Operation('create', op_id, op_begin, cluster=self) def update(self, timeout_seconds=None): """Update this cluster. @@ -430,7 +430,7 @@ def update(self, timeout_seconds=None): request_pb, timeout_seconds) op_id, op_begin = _process_operation(cluster_pb.current_operation) - return Operation('update', op_id, op_begin) + return Operation('update', op_id, op_begin, cluster=self) def delete(self, timeout_seconds=None): """Delete this cluster. @@ -504,7 +504,7 @@ def undelete(self, timeout_seconds=None): request_pb, timeout_seconds) op_id, op_begin = _process_operation(operation_pb2) - return Operation('undelete', op_id, op_begin) + return Operation('undelete', op_id, op_begin, cluster=self) def list_tables(self, timeout_seconds=None): """List the tables in this cluster. diff --git a/system_tests/run.py b/system_tests/run.py index 97bad29..d8b3978 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -120,13 +120,13 @@ def test_reload(self): def test_create_cluster(self): cluster_id = '%s-a' % (CLUSTER_ID,) cluster = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) - cluster.create() + operation = cluster.create() # Make sure this cluster gets deleted after the test case. self.clusters_to_delete.append(cluster) # We want to make sure the operation completes. time.sleep(2) - self.assertTrue(cluster.operation_finished()) + self.assertTrue(operation.finished()) # Create a new cluster instance and make sure it is the same. cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) @@ -139,11 +139,11 @@ def test_create_cluster(self): def test_update(self): curr_display_name = CLUSTER.display_name CLUSTER.display_name = 'Foo Bar Baz' - CLUSTER.update() + operation = CLUSTER.update() # We want to make sure the operation completes. time.sleep(2) - self.assertTrue(CLUSTER.operation_finished()) + self.assertTrue(operation.finished()) # Create a new cluster instance and make sure it is the same. cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID) @@ -156,11 +156,11 @@ def test_update(self): # Make sure to put the cluster back the way it was for the # other test cases. CLUSTER.display_name = curr_display_name - CLUSTER.update() + operation = CLUSTER.update() # We want to make sure the operation completes. time.sleep(2) - self.assertTrue(CLUSTER.operation_finished()) + self.assertTrue(operation.finished()) class TestTableAdminAPI(unittest2.TestCase): diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py index 803963a..6b6ea86 100644 --- a/system_tests/run_happybase.py +++ b/system_tests/run_happybase.py @@ -74,9 +74,9 @@ def set_cloud_bigtable_connection(): try: cluster.reload() except NetworkError: - cluster.create() + operation = cluster.create() count = 0 - while not cluster.operation_finished(): + while not operation.finished(): count += 1 if count > 5: raise NetworkError('Create cluster timed out') From c838b22ffe107236c5aad0eb1149fb8d3bccd1ac Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 8 Dec 2015 22:29:29 -0800 Subject: [PATCH 23/65] Removing unused type URL from type-map. --- gcloud_bigtable/_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 0512357..f9225e0 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -29,7 +29,6 @@ from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 from gcloud_bigtable._generated import duration_pb2 @@ -38,7 +37,6 @@ _CLUSTER_TYPE_URL = _ADMIN_TYPE_URL_BASE + 'Cluster' _CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' _TYPE_URL_MAP = { - _CLUSTER_TYPE_URL: data_pb2.Cluster, _CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata, _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata': ( messages_pb2.UndeleteClusterMetadata), From 128219ae2f6f07a4ef53f67c6502da7cc4fbc55b Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 8 Dec 2015 22:52:46 -0800 Subject: [PATCH 24/65] Updating docstring and failing tests from upstream. --- gcloud_bigtable/table.py | 4 ++-- gcloud_bigtable/test_cluster.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index d471e6b..4a270dc 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -167,8 +167,8 @@ def create(self, initial_split_keys=None, timeout_seconds=None): A create request returns a :class:`._generated.bigtable_table_data_pb2.Table` but we don't use this response. The proto definition allows for the inclusion of a - ``current_operation`` in the response, but in example usage so far, - it seems the Bigtable API does not return any operation. + ``current_operation`` in the response, but it does not appear that + the Cloud Bigtable API returns any operation. :type initial_split_keys: list :param initial_split_keys: (Optional) List of row keys that will be diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index 7792022..5f3f2fd 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -485,7 +485,8 @@ def test_create(self): # Create expected_result. op_id = 5678 op_begin = object() - expected_result = MUT.Operation('create', op_id, op_begin) + expected_result = MUT.Operation('create', op_id, op_begin, + cluster=cluster) # Perform the method and check the result. timeout_seconds = 578 @@ -539,7 +540,8 @@ def test_update(self): # Create expected_result. op_id = 5678 op_begin = object() - expected_result = MUT.Operation('update', op_id, op_begin) + expected_result = MUT.Operation('update', op_id, op_begin, + cluster=cluster) # We must create the cluster object with the client passed in. timeout_seconds = 9 @@ -615,7 +617,8 @@ def test_undelete(self): # Create expected_result. op_id = 5678 op_begin = object() - expected_result = MUT.Operation('undelete', op_id, op_begin) + expected_result = MUT.Operation('undelete', op_id, op_begin, + cluster=cluster) # Perform the method and check the result. timeout_seconds = 78 From 08b901eb8d18408afba1f02018751f4e427c9470 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 15 Dec 2015 22:55:26 -0800 Subject: [PATCH 25/65] Updating for upstream. In particular, ditching many convenience accessors and use of timeout_seconds. --- gcloud_bigtable/cluster.py | 131 ++++++++------------------------ gcloud_bigtable/table.py | 117 ++++++++-------------------- gcloud_bigtable/test_client.py | 2 +- gcloud_bigtable/test_cluster.py | 72 ++++++------------ gcloud_bigtable/test_table.py | 70 +++++++---------- 5 files changed, 112 insertions(+), 280 deletions(-) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index f937258..2035d44 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -34,6 +34,7 @@ _OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/zones/([^/]+)/' r'clusters/([a-z][-a-z0-9]*)/operations/' r'(?P\d+)$') +_DEFAULT_SERVE_NODES = 3 def _get_pb_property_value(message_pb, property_name): @@ -69,7 +70,7 @@ def _prepare_create_request(cluster): :rtype: :class:`.messages_pb2.CreateClusterRequest` :returns: The CreateCluster request object containing the cluster info. """ - zone_full_name = ('projects/' + cluster.project + + zone_full_name = ('projects/' + cluster._client.project + '/zones/' + cluster.zone) return messages_pb2.CreateClusterRequest( name=zone_full_name, @@ -97,8 +98,9 @@ def _process_operation(operation_pb): """ match = _OPERATION_NAME_RE.match(operation_pb.name) if match is None: - raise ValueError('Cluster create operation name was not in the ' - 'expected format.', operation_pb.name) + raise ValueError('Operation name was not in the expected ' + 'format after a cluster modification.', + operation_pb.name) operation_id = int(match.group('operation_id')) request_metadata = _parse_pb_any_to_native(operation_pb.metadata) @@ -147,14 +149,9 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) - def finished(self, timeout_seconds=None): + def finished(self): """Check if the operation has finished. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - :rtype: bool :returns: A boolean indicating if the current operation has completed. :raises: :class:`ValueError ` if the operation @@ -166,10 +163,9 @@ def finished(self, timeout_seconds=None): operation_name = ('operations/' + self._cluster.name + '/operations/%d' % (self.op_id,)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) - timeout_seconds = timeout_seconds or self._cluster.timeout_seconds - # We expact a `._generated.operations_pb2.Operation`. + # We expect a `._generated.operations_pb2.Operation`. operation_pb = self._cluster._client._operations_stub.GetOperation( - request_pb, timeout_seconds) + request_pb, self._cluster._client.timeout_seconds) if operation_pb.done: self._complete = True @@ -212,11 +208,11 @@ class Cluster(object): :type serve_nodes: int :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3. + Defaults to 3 (``_DEFAULT_SERVE_NODES``). """ def __init__(self, zone, cluster_id, client, - display_name=None, serve_nodes=3): + display_name=None, serve_nodes=_DEFAULT_SERVE_NODES): self.zone = zone self.cluster_id = cluster_id self.display_name = display_name or cluster_id @@ -276,38 +272,11 @@ def copy(self): :rtype: :class:`.Cluster` :returns: A copy of the current cluster. """ - new_client = self.client.copy() + new_client = self._client.copy() return self.__class__(self.zone, self.cluster_id, new_client, display_name=self.display_name, serve_nodes=self.serve_nodes) - @property - def client(self): - """Getter for cluster's client. - - :rtype: :class:`.client.Client` - :returns: The client stored on the cluster. - """ - return self._client - - @property - def project(self): - """Getter for cluster's project ID. - - :rtype: str - :returns: The project ID for the cluster (is stored on the client). - """ - return self._client.project - - @property - def timeout_seconds(self): - """Getter for cluster's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default stored on the cluster's client. - """ - return self._client.timeout_seconds - @property def name(self): """Cluster name used in requests. @@ -323,7 +292,7 @@ def name(self): :rtype: str :returns: The cluster name. """ - return (self.client.project_name + '/zones/' + self.zone + + return (self._client.project_name + '/zones/' + self.zone + '/clusters/' + self.cluster_id) def __eq__(self, other): @@ -337,30 +306,23 @@ def __eq__(self, other): # settings but different clients can't be used in the same way. return (other.zone == self.zone and other.cluster_id == self.cluster_id and - other.client == self.client) + other._client == self._client) def __ne__(self, other): return not self.__eq__(other) - def reload(self, timeout_seconds=None): - """Reload the metadata for this cluster. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ + def reload(self): + """Reload the metadata for this cluster.""" request_pb = messages_pb2.GetClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = self.client._cluster_stub.GetCluster(request_pb, - timeout_seconds) + cluster_pb = self._client._cluster_stub.GetCluster( + request_pb, self._client.timeout_seconds) # NOTE: _update_from_pb does not check that the project, zone and # cluster ID on the response match the request. self._update_from_pb(cluster_pb) - def create(self, timeout_seconds=None): + def create(self): """Create this cluster. .. note:: @@ -377,25 +339,19 @@ def create(self, timeout_seconds=None): before calling :meth:`create`. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - :rtype: :class:`Operation` :returns: The long-running operation corresponding to the create operation. """ request_pb = _prepare_create_request(self) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect an `operations_pb2.Operation`. - cluster_pb = self.client._cluster_stub.CreateCluster( - request_pb, timeout_seconds) + cluster_pb = self._client._cluster_stub.CreateCluster( + request_pb, self._client.timeout_seconds) op_id, op_begin = _process_operation(cluster_pb.current_operation) return Operation('create', op_id, op_begin, cluster=self) - def update(self, timeout_seconds=None): + def update(self): """Update this cluster. .. note:: @@ -410,11 +366,6 @@ def update(self, timeout_seconds=None): before calling :meth:`update`. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - :rtype: :class:`Operation` :returns: The long-running operation corresponding to the update operation. @@ -424,15 +375,14 @@ def update(self, timeout_seconds=None): display_name=self.display_name, serve_nodes=self.serve_nodes, ) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = self.client._cluster_stub.UpdateCluster( - request_pb, timeout_seconds) + cluster_pb = self._client._cluster_stub.UpdateCluster( + request_pb, self._client.timeout_seconds) op_id, op_begin = _process_operation(cluster_pb.current_operation) return Operation('update', op_id, op_begin, cluster=self) - def delete(self, timeout_seconds=None): + def delete(self): """Delete this cluster. Marks a cluster and all of its tables for permanent deletion in 7 days. @@ -456,18 +406,13 @@ def delete(self, timeout_seconds=None): * The cluster and **all of its tables** will immediately and irrevocably disappear from the API, and their data will be permanently deleted. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. """ request_pb = messages_pb2.DeleteClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `._generated.empty_pb2.Empty` - self.client._cluster_stub.DeleteCluster(request_pb, timeout_seconds) + self._client._cluster_stub.DeleteCluster( + request_pb, self._client.timeout_seconds) - def undelete(self, timeout_seconds=None): + def undelete(self): """Undelete this cluster. Cancels the scheduled deletion of an cluster and begins preparing it to @@ -488,42 +433,30 @@ def undelete(self, timeout_seconds=None): * Billing for the cluster's resources will resume. * All tables within the cluster will be available. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - :rtype: :class:`Operation` :returns: The long-running operation corresponding to the undelete operation. """ request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `._generated.operations_pb2.Operation` - operation_pb2 = self.client._cluster_stub.UndeleteCluster( - request_pb, timeout_seconds) + operation_pb2 = self._client._cluster_stub.UndeleteCluster( + request_pb, self._client.timeout_seconds) op_id, op_begin = _process_operation(operation_pb2) return Operation('undelete', op_id, op_begin, cluster=self) - def list_tables(self, timeout_seconds=None): + def list_tables(self): """List the tables in this cluster. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - :rtype: list of :class:`Table ` :returns: The list of tables owned by the cluster. :raises: :class:`ValueError ` if one of the returned tables has a name that is not of the expected format. """ request_pb = table_messages_pb2.ListTablesRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `table_messages_pb2.ListTablesResponse` - table_list_pb = self.client._table_stub.ListTables(request_pb, - timeout_seconds) + table_list_pb = self._client._table_stub.ListTables( + request_pb, self._client.timeout_seconds) result = [] for table_pb in table_list_pb.tables: diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index 4a270dc..27012fa 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -61,33 +61,6 @@ def __init__(self, table_id, cluster): self.table_id = table_id self._cluster = cluster - @property - def cluster(self): - """Getter for table's cluster. - - :rtype: :class:`.cluster.Cluster` - :returns: The cluster stored on the table. - """ - return self._cluster - - @property - def client(self): - """Getter for table's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this table. - """ - return self.cluster.client - - @property - def timeout_seconds(self): - """Getter for table's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default stored on the table's client. - """ - return self._cluster.timeout_seconds - @property def name(self): """Table name used in requests. @@ -104,7 +77,7 @@ def name(self): :rtype: str :returns: The table name. """ - return self.cluster.name + '/tables/' + self.table_id + return self._cluster.name + '/tables/' + self.table_id def column_family(self, column_family_id, gc_rule=None): """Factory to create a column family associated with this table. @@ -146,12 +119,12 @@ def __eq__(self, other): if not isinstance(other, self.__class__): return False return (other.table_id == self.table_id and - other.cluster == self.cluster) + other._cluster == self._cluster) def __ne__(self, other): return not self.__eq__(other) - def create(self, initial_split_keys=None, timeout_seconds=None): + def create(self, initial_split_keys=None): """Creates this table. .. note:: @@ -178,21 +151,17 @@ def create(self, initial_split_keys=None, timeout_seconds=None): ``"s1"`` and ``"s2"``, three tablets will be created, spanning the key ranges: ``[, s1)``, ``[s1, s2)``, ``[s2, )``. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. """ request_pb = messages_pb2.CreateTableRequest( initial_split_keys=initial_split_keys or [], - name=self.cluster.name, + name=self._cluster.name, table_id=self.table_id, ) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._cluster._client # We expect a `._generated.bigtable_table_data_pb2.Table` - self.client._table_stub.CreateTable(request_pb, timeout_seconds) + client._table_stub.CreateTable(request_pb, client.timeout_seconds) - def rename(self, new_table_id, timeout_seconds=None): + def rename(self, new_table_id): """Rename this table. .. note:: @@ -202,7 +171,7 @@ def rename(self, new_table_id, timeout_seconds=None): .. note:: - The Bigtable Table Admin API currently returns + The Bigtable Table Admin API currently (``v1``) returns ``BigtableTableService.RenameTable is not yet implemented`` @@ -211,40 +180,27 @@ def rename(self, new_table_id, timeout_seconds=None): :type new_table_id: str :param new_table_id: The new name table ID. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. """ request_pb = messages_pb2.RenameTableRequest( name=self.name, new_id=new_table_id, ) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._cluster._client # We expect a `._generated.empty_pb2.Empty` - self.client._table_stub.RenameTable(request_pb, timeout_seconds) + client._table_stub.RenameTable(request_pb, client.timeout_seconds) self.table_id = new_table_id - def delete(self, timeout_seconds=None): - """Delete this table. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - """ + def delete(self): + """Delete this table.""" request_pb = messages_pb2.DeleteTableRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._cluster._client # We expect a `._generated.empty_pb2.Empty` - self.client._table_stub.DeleteTable(request_pb, timeout_seconds) + client._table_stub.DeleteTable(request_pb, client.timeout_seconds) - def list_column_families(self, timeout_seconds=None): + def list_column_families(self): """Check if this table exists. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - :rtype: dictionary with string as keys and :class:`.column_family.ColumnFamily` as values :returns: List of column families attached to this table. @@ -253,10 +209,10 @@ def list_column_families(self, timeout_seconds=None): name from the column family ID. """ request_pb = messages_pb2.GetTableRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._cluster._client # We expect a `._generated.bigtable_table_data_pb2.Table` - table_pb = self.client._table_stub.GetTable(request_pb, - timeout_seconds) + table_pb = client._table_stub.GetTable(request_pb, + client.timeout_seconds) result = {} for column_family_id, value_pb in table_pb.column_families.items(): @@ -270,7 +226,7 @@ def list_column_families(self, timeout_seconds=None): result[column_family_id] = column_family return result - def read_row(self, row_key, filter_=None, timeout_seconds=None): + def read_row(self, row_key, filter_=None): """Read a single row from this table. :type row_key: bytes @@ -282,10 +238,6 @@ def read_row(self, row_key, filter_=None, timeout_seconds=None): :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - :rtype: :class:`.PartialRowData`, :data:`NoneType ` :returns: The contents of the row if any chunks were returned in the response, otherwise :data:`None`. @@ -294,9 +246,9 @@ def read_row(self, row_key, filter_=None, timeout_seconds=None): """ request_pb = _create_row_request(self.name, row_key=row_key, filter_=filter_) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.ReadRows(request_pb, - timeout_seconds) + client = self._cluster._client + response_iterator = client._data_stub.ReadRows(request_pb, + client.timeout_seconds) # We expect an iterator of `data_messages_pb2.ReadRowsResponse` result = PartialRowData(row_key) for read_rows_response in response_iterator: @@ -311,8 +263,7 @@ def read_row(self, row_key, filter_=None, timeout_seconds=None): return result def read_rows(self, start_key=None, end_key=None, - allow_row_interleaving=None, limit=None, filter_=None, - timeout_seconds=None): + allow_row_interleaving=None, limit=None, filter_=None): """Read rows from this table. :type start_key: bytes @@ -353,10 +304,6 @@ def read_rows(self, start_key=None, end_key=None, more than N rows. However, only N ``commit_row`` chunks will be sent. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - :rtype: :class:`.PartialRowsData` :returns: A :class:`.PartialRowsData` convenience wrapper for consuming the streamed results. @@ -364,13 +311,13 @@ def read_rows(self, start_key=None, end_key=None, request_pb = _create_row_request( self.name, start_key=start_key, end_key=end_key, filter_=filter_, allow_row_interleaving=allow_row_interleaving, limit=limit) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.ReadRows(request_pb, - timeout_seconds) + client = self._cluster._client + response_iterator = client._data_stub.ReadRows(request_pb, + client.timeout_seconds) # We expect an iterator of `data_messages_pb2.ReadRowsResponse` return PartialRowsData(response_iterator) - def sample_row_keys(self, timeout_seconds=None): + def sample_row_keys(self): """Read a sample of row keys in the table. The returned row keys will delimit contiguous sections of the table of @@ -396,10 +343,6 @@ def sample_row_keys(self, timeout_seconds=None): samples would require space roughly equal to the difference in their ``offset_bytes`` fields. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - :rtype: :class:`grpc.framework.alpha._reexport._CancellableIterator` :returns: A cancel-able iterator. Can be consumed by calling ``next()`` or by casting to a :class:`list` and can be cancelled by @@ -407,9 +350,9 @@ def sample_row_keys(self, timeout_seconds=None): """ request_pb = data_messages_pb2.SampleRowKeysRequest( table_name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.SampleRowKeys( - request_pb, timeout_seconds) + client = self._cluster._client + response_iterator = client._data_stub.SampleRowKeys( + request_pb, client.timeout_seconds) return response_iterator diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index a2eb0af..d3803a9 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -774,7 +774,7 @@ def test_cluster_factory(self): cluster_id = 'cluster-id' cluster = client.cluster(zone, cluster_id) self.assertTrue(isinstance(cluster, Cluster)) - self.assertTrue(cluster.client is client) + self.assertTrue(cluster._client is client) self.assertEqual(cluster.zone, zone) self.assertEqual(cluster.cluster_id, cluster_id) diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index 5f3f2fd..4eebb6d 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -198,7 +198,7 @@ def _finished_helper(self, done): begin = datetime.datetime(2015, 10, 22, 1, 1) timeout_seconds = 1 - client = _Client(project) + client = _Client(project, timeout_seconds=timeout_seconds) cluster = Cluster(zone, cluster_id, client) operation = self._makeOne(op_type, op_id, begin, cluster=cluster) @@ -218,7 +218,7 @@ def _finished_helper(self, done): expected_result = done # Perform the method and check the result. - result = operation.finished(timeout_seconds=timeout_seconds) + result = operation.finished() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -289,33 +289,6 @@ def test_copy(self): self.assertFalse(cluster is new_cluster) self.assertEqual(cluster, new_cluster) - def test_client_getter(self): - client = object() - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertTrue(cluster.client is client) - - def test_project_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.project, PROJECT) - - def test_timeout_seconds_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - timeout_seconds = 77 - client = Client(project=PROJECT, credentials=credentials, - timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.timeout_seconds, timeout_seconds) - def test_name_property(self): from gcloud_bigtable._testing import _MockWithAttachedMethods from gcloud_bigtable.client import Client @@ -359,7 +332,7 @@ def test_from_pb_success(self): klass = self._getTargetClass() cluster = klass.from_pb(cluster_pb, client) self.assertTrue(isinstance(cluster, klass)) - self.assertEqual(cluster.client, client) + self.assertEqual(cluster._client, client) self.assertEqual(cluster.zone, ZONE) self.assertEqual(cluster.cluster_id, CLUSTER_ID) @@ -429,7 +402,8 @@ def test_reload(self): bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client(PROJECT) + timeout_seconds = 123 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) cluster = self._makeOne(ZONE, CLUSTER_ID, client) # Create request_pb @@ -450,8 +424,7 @@ def test_reload(self): expected_result = None # reload() has no return value. # Perform the method and check the result. - timeout_seconds = 123 - result = cluster.reload(timeout_seconds=timeout_seconds) + result = cluster.reload() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetCluster', @@ -468,7 +441,8 @@ def test_create(self): from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT - client = _Client(PROJECT) + timeout_seconds = 578 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) cluster = self._makeOne(ZONE, CLUSTER_ID, client) # Create request_pb. Just a mock since we monkey patch @@ -489,12 +463,11 @@ def test_create(self): cluster=cluster) # Perform the method and check the result. - timeout_seconds = 578 mock_prepare_create_request = _MockCalled(request_pb) mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _prepare_create_request=mock_prepare_create_request, _process_operation=mock_process_operation): - result = cluster.create(timeout_seconds=timeout_seconds) + result = cluster.create() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -514,7 +487,8 @@ def test_update(self): from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT - client = _Client(PROJECT) + timeout_seconds = 9 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) serve_nodes = 81 display_name = 'display_name' cluster = self._makeOne(ZONE, CLUSTER_ID, client, @@ -544,11 +518,10 @@ def test_update(self): cluster=cluster) # We must create the cluster object with the client passed in. - timeout_seconds = 9 mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _process_operation=mock_process_operation): - result = cluster.update(timeout_seconds=timeout_seconds) + result = cluster.update() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -564,7 +537,8 @@ def test_delete(self): from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock - client = _Client(PROJECT) + timeout_seconds = 57 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) cluster = self._makeOne(ZONE, CLUSTER_ID, client) # Create request_pb @@ -582,8 +556,7 @@ def test_delete(self): expected_result = None # delete() has no return value. # Perform the method and check the result. - timeout_seconds = 57 - result = cluster.delete(timeout_seconds=timeout_seconds) + result = cluster.delete() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteCluster', @@ -600,7 +573,8 @@ def test_undelete(self): from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT - client = _Client(PROJECT) + timeout_seconds = 78 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) cluster = self._makeOne(ZONE, CLUSTER_ID, client) # Create request_pb @@ -621,11 +595,10 @@ def test_undelete(self): cluster=cluster) # Perform the method and check the result. - timeout_seconds = 78 mock_process_operation = _MockCalled((op_id, op_begin)) with _Monkey(MUT, _process_operation=mock_process_operation): - result = cluster.undelete(timeout_seconds=timeout_seconds) + result = cluster.undelete() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -642,7 +615,8 @@ def _list_tables_helper(self, table_id, table_name=None): bigtable_table_service_messages_pb2 as table_messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client(PROJECT) + timeout_seconds = 45 + client = _Client(PROJECT, timeout_seconds=timeout_seconds) cluster = self._makeOne(ZONE, CLUSTER_ID, client) # Create request_ @@ -666,8 +640,7 @@ def _list_tables_helper(self, table_id, table_name=None): expected_result = [expected_table] # Perform the method and check the result. - timeout_seconds = 45 - result = cluster.list_tables(timeout_seconds=timeout_seconds) + result = cluster.list_tables() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListTables', @@ -698,9 +671,10 @@ class _Client(object): operations_stub = None table_stub = None - def __init__(self, project): + def __init__(self, project, timeout_seconds=None): self.project = project self.project_name = 'projects/' + project + self.timeout_seconds = timeout_seconds def copy(self): import copy as copy_module diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py index fa9b551..17781c6 100644 --- a/gcloud_bigtable/test_table.py +++ b/gcloud_bigtable/test_table.py @@ -37,23 +37,6 @@ def test_constructor(self): self.assertEqual(table.table_id, TABLE_ID) self.assertTrue(table._cluster is cluster) - def test_cluster_getter(self): - cluster = object() - table = self._makeOne(TABLE_ID, cluster) - self.assertTrue(table.cluster is cluster) - - def test_client_getter(self): - client = object() - cluster = _Cluster(None, client=client) - table = self._makeOne(TABLE_ID, cluster) - self.assertTrue(table.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 1001 - cluster = _Cluster(None, timeout_seconds=timeout_seconds) - table = self._makeOne(TABLE_ID, cluster) - self.assertEqual(table.timeout_seconds, timeout_seconds) - def test_name_property(self): cluster_name = 'cluster_name' cluster = _Cluster(cluster_name) @@ -117,7 +100,8 @@ def _create_test_helper(self, initial_split_keys): bigtable_table_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 150 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -140,9 +124,7 @@ def _create_test_helper(self, initial_split_keys): expected_result = None # create() has no return value. # Perform the method and check the result. - timeout_seconds = 150 - result = table.create(initial_split_keys=initial_split_keys, - timeout_seconds=timeout_seconds) + result = table.create(initial_split_keys=initial_split_keys) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateTable', @@ -167,7 +149,8 @@ def test_rename(self): new_table_id = 'new_table_id' self.assertNotEqual(new_table_id, TABLE_ID) - client = _Client() + timeout_seconds = 97 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -190,8 +173,7 @@ def test_rename(self): expected_result = None # rename() has no return value. # Perform the method and check the result. - timeout_seconds = 97 - result = table.rename(new_table_id, timeout_seconds=timeout_seconds) + result = table.rename(new_table_id) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'RenameTable', @@ -205,7 +187,8 @@ def test_delete(self): from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 871 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -225,8 +208,7 @@ def test_delete(self): expected_result = None # delete() has no return value. # Perform the method and check the result. - timeout_seconds = 871 - result = table.delete(timeout_seconds=timeout_seconds) + result = table.delete() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteTable', @@ -242,7 +224,8 @@ def _list_column_families_helper(self, column_family_name=None): from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable.column_family import ColumnFamily - client = _Client() + timeout_seconds = 502 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -271,8 +254,7 @@ def _list_column_families_helper(self, column_family_name=None): } # Perform the method and check the result. - timeout_seconds = 502 - result = table.list_column_families(timeout_seconds=timeout_seconds) + result = table.list_column_families() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'GetTable', @@ -298,7 +280,8 @@ def _read_row_helper(self, chunks): from gcloud_bigtable.row_data import PartialRowData from gcloud_bigtable import table as MUT - client = _Client() + timeout_seconds = 596 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -327,10 +310,8 @@ def _read_row_helper(self, chunks): # Perform the method and check the result. filter_obj = object() - timeout_seconds = 596 with _Monkey(MUT, _create_row_request=mock_create_row_request): - result = table.read_row(row_key, filter_=filter_obj, - timeout_seconds=timeout_seconds) + result = table.read_row(row_key, filter_=filter_obj) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -371,7 +352,8 @@ def test_read_rows(self): from gcloud_bigtable.row_data import PartialRowsData from gcloud_bigtable import table as MUT - client = _Client() + timeout_seconds = 1111 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -396,12 +378,10 @@ def test_read_rows(self): filter_obj = object() allow_row_interleaving = True limit = 22 - timeout_seconds = 1111 with _Monkey(MUT, _create_row_request=mock_create_row_request): result = table.read_rows( start_key=start_key, end_key=end_key, filter_=filter_obj, - allow_row_interleaving=allow_row_interleaving, limit=limit, - timeout_seconds=timeout_seconds) + allow_row_interleaving=allow_row_interleaving, limit=limit) self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( @@ -424,7 +404,8 @@ def test_sample_row_keys(self): bigtable_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 1333 + client = _Client(timeout_seconds=timeout_seconds) cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID) cluster = _Cluster(cluster_name, client=client) @@ -444,8 +425,7 @@ def test_sample_row_keys(self): expected_result = response_iterator # Perform the method and check the result. - timeout_seconds = 1333 - result = table.sample_row_keys(timeout_seconds=timeout_seconds) + result = table.sample_row_keys() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'SampleRowKeys', @@ -583,10 +563,12 @@ class _Client(object): operations_stub = None table_stub = None + def __init__(self, timeout_seconds=None): + self.timeout_seconds = timeout_seconds + class _Cluster(object): - def __init__(self, name, client=None, timeout_seconds=None): + def __init__(self, name, client=None): self.name = name - self.client = client - self.timeout_seconds = timeout_seconds + self._client = client From f168b909ace0ed3e5550cbbf1898f5c9a2a12a48 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 15 Dec 2015 23:00:55 -0800 Subject: [PATCH 26/65] Updating accessor code so system tests update. Based on removing many @property's since not used in upstream. --- gcloud_bigtable/column_family.py | 4 ++-- gcloud_bigtable/happybase/connection.py | 4 ++-- gcloud_bigtable/row.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index c85d2d5..aab9c82 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -198,7 +198,7 @@ def client(self): :rtype: :class:`.client.Client` :returns: The client that owns this column family. """ - return self.table.client + return self.table._cluster._client @property def timeout_seconds(self): @@ -207,7 +207,7 @@ def timeout_seconds(self): :rtype: int :returns: The timeout seconds default. """ - return self.table.timeout_seconds + return self.table._cluster._client.timeout_seconds @property def name(self): diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 95dd525..ddbfc39 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -245,7 +245,7 @@ def open(self): :class:`.Client` bound to the :class:`.Cluster` owned by this connection. """ - self._cluster.client.start() + self._cluster._client.start() def close(self): """Close the underlying transport to Cloud Bigtable. @@ -254,7 +254,7 @@ def close(self): :class:`.Client` bound to the :class:`.Cluster` owned by this connection. """ - self._cluster.client.stop() + self._cluster._client.stop() def __del__(self): try: diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index c070dfc..917fd0e 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -113,7 +113,7 @@ def client(self): :rtype: :class:`.client.Client` :returns: The client that owns this row. """ - return self.table.client + return self.table._cluster._client @property def timeout_seconds(self): @@ -122,7 +122,7 @@ def timeout_seconds(self): :rtype: int :returns: The timeout seconds default. """ - return self.table.timeout_seconds + return self.table._cluster._client.timeout_seconds def _get_mutations(self, state=None): """Gets the list of mutations for a given state. From 5c6680d03523820b75872ecfdd8247019d359aad Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 15 Dec 2015 23:13:17 -0800 Subject: [PATCH 27/65] Updating unit tests that failed after fixing system tests. --- gcloud_bigtable/happybase/test_connection.py | 28 ++++++++++---------- gcloud_bigtable/happybase/test_pool.py | 2 +- gcloud_bigtable/test_column_family.py | 14 ++++++++-- gcloud_bigtable/test_row.py | 14 ++++++++-- pylintrc_default | 2 +- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index 397ecea..d126bb7 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -154,10 +154,10 @@ def _makeOne(self, *args, **kwargs): def test_constructor_defaults(self): cluster = _Cluster() # Avoid implicit environ check. - self.assertEqual(cluster.client.start_calls, 0) + self.assertEqual(cluster._client.start_calls, 0) connection = self._makeOne(cluster=cluster) - self.assertEqual(cluster.client.start_calls, 1) - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.start_calls, 1) + self.assertEqual(cluster._client.stop_calls, 0) self.assertEqual(connection._cluster, cluster) self.assertEqual(connection.table_prefix, None) @@ -243,25 +243,25 @@ def test_constructor_with_timeout_and_cluster(self): def test_open(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.start_calls, 0) + self.assertEqual(cluster._client.start_calls, 0) connection.open() - self.assertEqual(cluster.client.start_calls, 1) - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.start_calls, 1) + self.assertEqual(cluster._client.stop_calls, 0) def test_close(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.stop_calls, 0) connection.close() - self.assertEqual(cluster.client.stop_calls, 1) - self.assertEqual(cluster.client.start_calls, 0) + self.assertEqual(cluster._client.stop_calls, 1) + self.assertEqual(cluster._client.start_calls, 0) def test___del__good_initialization(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.stop_calls, 0) connection.__del__() - self.assertEqual(cluster.client.stop_calls, 1) + self.assertEqual(cluster._client.stop_calls, 1) def test___del__bad_initialization(self): cluster = _Cluster() # Avoid implicit environ check. @@ -269,9 +269,9 @@ def test___del__bad_initialization(self): # Fake that initialization failed. del connection._initialized - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.stop_calls, 0) connection.__del__() - self.assertEqual(cluster.client.stop_calls, 0) + self.assertEqual(cluster._client.stop_calls, 0) def test__table_name_with_prefix_set(self): table_prefix = 'table-prefix' @@ -529,7 +529,7 @@ class _Cluster(object): def __init__(self, copies=(), list_tables_result=()): self.copies = list(copies) # Included to support Connection.__del__ - self.client = _Client() + self._client = _Client() self.list_tables_result = list_tables_result def copy(self): diff --git a/gcloud_bigtable/happybase/test_pool.py b/gcloud_bigtable/happybase/test_pool.py index 6791c57..d6142e7 100644 --- a/gcloud_bigtable/happybase/test_pool.py +++ b/gcloud_bigtable/happybase/test_pool.py @@ -227,7 +227,7 @@ class _Cluster(object): def __init__(self, copies=()): self.copies = list(copies) # Included to support Connection.__del__ - self.client = _Client() + self._client = _Client() def copy(self): if self.copies: diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py index 66e7282..e2ae9f4 100644 --- a/gcloud_bigtable/test_column_family.py +++ b/gcloud_bigtable/test_column_family.py @@ -302,7 +302,8 @@ def test_client_getter(self): def test_timeout_seconds_getter(self): timeout_seconds = 889 - table = _Table(None, timeout_seconds=timeout_seconds) + client = _Client(timeout_seconds=timeout_seconds) + table = _Table(None, client=client) column_family = self._makeOne(COLUMN_FAMILY_ID, table) self.assertEqual(column_family.timeout_seconds, timeout_seconds) @@ -573,10 +574,19 @@ class _Client(object): operations_stub = None table_stub = None + def __init__(self, timeout_seconds=None): + self.timeout_seconds = timeout_seconds + + +class _Cluster(object): + + def __init__(self, client=None): + self._client = client + class _Table(object): def __init__(self, name, client=None, timeout_seconds=None): self.name = name - self.client = client + self._cluster = _Cluster(client) self.timeout_seconds = timeout_seconds diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index cc5556e..c965c23 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -94,7 +94,8 @@ def test_client_getter(self): def test_timeout_seconds_getter(self): timeout_seconds = 889 - table = _Table(None, timeout_seconds=timeout_seconds) + client = _Client(timeout_seconds=timeout_seconds) + table = _Table(None, client=client) row = self._makeOne(ROW_KEY, table) self.assertEqual(row.timeout_seconds, timeout_seconds) @@ -1526,10 +1527,19 @@ class _Client(object): data_stub = None + def __init__(self, timeout_seconds=None): + self.timeout_seconds = timeout_seconds + + +class _Cluster(object): + + def __init__(self, client=None): + self._client = client + class _Table(object): def __init__(self, name, client=None, timeout_seconds=None): self.name = name - self.client = client + self._cluster = _Cluster(client) self.timeout_seconds = timeout_seconds diff --git a/pylintrc_default b/pylintrc_default index 6fa9050..6e33cbc 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -205,7 +205,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1540 +max-module-lines=1545 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). From f70871b94e641d7bc21bfb02771e76e5af2a659d Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 15:14:07 -0800 Subject: [PATCH 28/65] Updating Makefile with upstream. --- Makefile | 85 ++- gcloud_bigtable/_generated/_annotations.proto | 29 + gcloud_bigtable/_generated/_any.proto | 87 +++ .../_generated/_bigtable_cluster_data.proto | 94 ++++ .../_bigtable_cluster_service.proto | 129 +++++ .../_bigtable_cluster_service_messages.proto | 134 +++++ .../_generated/_bigtable_data.proto | 500 ++++++++++++++++++ .../_generated/_bigtable_service.proto | 66 +++ .../_bigtable_service_messages.proto | 177 +++++++ .../_generated/_bigtable_table_data.proto | 125 +++++ .../_generated/_bigtable_table_service.proto | 74 +++ .../_bigtable_table_service_messages.proto | 101 ++++ gcloud_bigtable/_generated/_duration.proto | 78 +++ gcloud_bigtable/_generated/_empty.proto | 34 ++ gcloud_bigtable/_generated/_http.proto | 127 +++++ gcloud_bigtable/_generated/_operations.proto | 144 +++++ gcloud_bigtable/_generated/_status.proto | 88 +++ gcloud_bigtable/_generated/_timestamp.proto | 89 ++++ scripts/rewrite_imports.py | 9 +- 19 files changed, 2125 insertions(+), 45 deletions(-) create mode 100644 gcloud_bigtable/_generated/_annotations.proto create mode 100644 gcloud_bigtable/_generated/_any.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_data.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_service.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_data.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_service.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_service_messages.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_table_data.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_table_service.proto create mode 100644 gcloud_bigtable/_generated/_bigtable_table_service_messages.proto create mode 100644 gcloud_bigtable/_generated/_duration.proto create mode 100644 gcloud_bigtable/_generated/_empty.proto create mode 100644 gcloud_bigtable/_generated/_http.proto create mode 100644 gcloud_bigtable/_generated/_operations.proto create mode 100644 gcloud_bigtable/_generated/_status.proto create mode 100644 gcloud_bigtable/_generated/_timestamp.proto diff --git a/Makefile b/Makefile index 11cb68b..1254d35 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ GENERATED_DIR=$(shell pwd)/generated_python -FINAL_DIR=gcloud_bigtable/_generated +FINAL_DIR=$(shell pwd)/gcloud_bigtable/_generated GRPC_PLUGIN=grpc_python_plugin +PROTOC_CMD=protoc +PROTOS_DIR=$(shell pwd)/cloud-bigtable-client/bigtable-protos/src/main/proto help: @echo 'Makefile for a gcloud-python-bigtable ' @@ -13,59 +15,54 @@ generate: [ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client cd cloud-bigtable-client && git pull origin master mkdir -p $(GENERATED_DIR) - # Data API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) google/bigtable/v1/*.proto - mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) - # Cluster API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ + # Generate all *_pb2.py files. + $(PROTOC_CMD) \ + --proto_path=$(PROTOS_DIR) \ --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/bigtable/admin/cluster/v1/*.proto + --python_out=$(GENERATED_DIR) \ + --grpc_out=$(GENERATED_DIR) \ + $(PROTOS_DIR)/google/bigtable/v1/*.proto \ + $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto \ + $(PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto \ + $(PROTOS_DIR)/google/api/*.proto \ + $(PROTOS_DIR)/google/protobuf/any.proto \ + $(PROTOS_DIR)/google/protobuf/duration.proto \ + $(PROTOS_DIR)/google/protobuf/empty.proto \ + $(PROTOS_DIR)/google/protobuf/timestamp.proto \ + $(PROTOS_DIR)/google/longrunning/operations.proto \ + $(PROTOS_DIR)/google/rpc/status.proto + # Move the newly generated *_pb2.py files into our library. + mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(FINAL_DIR) - # Table API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/bigtable/admin/table/v1/*.proto mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(FINAL_DIR) - # Auxiliary protos - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/api/*.proto mv $(GENERATED_DIR)/google/api/* $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/any.proto mv $(GENERATED_DIR)/google/protobuf/any_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/duration.proto mv $(GENERATED_DIR)/google/protobuf/duration_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/empty.proto mv $(GENERATED_DIR)/google/protobuf/empty_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/timestamp.proto mv $(GENERATED_DIR)/google/protobuf/timestamp_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/longrunning/operations.proto mv $(GENERATED_DIR)/google/longrunning/operations_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/rpc/status.proto mv $(GENERATED_DIR)/google/rpc/status_pb2.py $(FINAL_DIR) + # Remove all existing *.proto files before we replace + rm -f $(FINAL_DIR)/*.proto + # Copy over the *.proto files into our library. + cp $(PROTOS_DIR)/google/bigtable/v1/*.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/api/*.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/protobuf/any.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/protobuf/duration.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/protobuf/empty.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/protobuf/timestamp.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/longrunning/operations.proto $(FINAL_DIR) + cp $(PROTOS_DIR)/google/rpc/status.proto $(FINAL_DIR) + # Rename all *.proto files in our library with an + # underscore and remove executable bit. + cd $(FINAL_DIR) && \ + for filename in *.proto; do \ + chmod -x $$filename ; \ + mv $$filename _$$filename ; \ + done + # Rewrite the imports in the generated *_pb2.py files. python scripts/rewrite_imports.py check_generate: diff --git a/gcloud_bigtable/_generated/_annotations.proto b/gcloud_bigtable/_generated/_annotations.proto new file mode 100644 index 0000000..cbd18b8 --- /dev/null +++ b/gcloud_bigtable/_generated/_annotations.proto @@ -0,0 +1,29 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} diff --git a/gcloud_bigtable/_generated/_any.proto b/gcloud_bigtable/_generated/_any.proto new file mode 100644 index 0000000..7c79e57 --- /dev/null +++ b/gcloud_bigtable/_generated/_any.proto @@ -0,0 +1,87 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "AnyProto"; +option java_package = "com.google.protobuf"; + + +// `Any` contains an arbitrary serialized message along with a URL +// that describes the type of the serialized message. +// +// The proto runtimes and/or compiler will eventually +// provide utilities to pack/unpack Any values (projected Q1/15). +// +// # JSON +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the the `@type` +// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name whose content describes the type of the + // serialized message. + // + // For URLs which use the schema `http`, `https`, or no schema, the + // following restrictions and interpretations apply: + // + // * If no schema is provided, `https` is assumed. + // * The last segment of the URL's path must represent the fully + // qualified name of the type (as in `path/google.protobuf.Duration`). + // * An HTTP GET on the URL must yield a [google.protobuf.Type][google.protobuf.Type] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Schemas other than `http`, `https` (or the empty schema) might be + // used with implementation specific semantics. + // + // Types originating from the `google.*` package + // namespace should use `type.googleapis.com/full.type.name` (without + // schema and path). A type service will eventually become available which + // serves those URLs (projected Q2/15). + string type_url = 1; + + // Must be valid serialized data of the above specified type. + bytes value = 2; +} diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_data.proto b/gcloud_bigtable/_generated/_bigtable_cluster_data.proto new file mode 100644 index 0000000..cbebe18 --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_cluster_data.proto @@ -0,0 +1,94 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + +import "google/api/annotations.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterDataProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// A physical location in which a particular project can allocate Cloud BigTable +// resources. +message Zone { + // Possible states of a zone. + enum Status { + // The state of the zone is unknown or unspecified. + UNKNOWN = 0; + + // The zone is in a good state. + OK = 1; + + // The zone is down for planned maintenance. + PLANNED_MAINTENANCE = 2; + + // The zone is down for emergency or unplanned maintenance. + EMERGENCY_MAINENANCE = 3; + } + + // A permanent unique identifier for the zone. + // Values are of the form projects//zones/[a-z][-a-z0-9]* + string name = 1; + + // The name of this zone as it appears in UIs. + string display_name = 2; + + // The current state of this zone. + Status status = 3; +} + +// An isolated set of Cloud BigTable resources on which tables can be hosted. +message Cluster { + // A permanent unique identifier for the cluster. For technical reasons, the + // zone in which the cluster resides is included here. + // Values are of the form + // projects//zones//clusters/[a-z][-a-z0-9]* + string name = 1; + + // If this cluster has been deleted, the time at which its backup will + // be irrevocably destroyed. Omitted otherwise. + // This cannot be set directly, only through DeleteCluster. + google.protobuf.Timestamp delete_time = 2; + + // The operation currently running on the cluster, if any. + // This cannot be set directly, only through CreateCluster, UpdateCluster, + // or UndeleteCluster. Calls to these methods will be rejected if + // "current_operation" is already set. + google.longrunning.Operation current_operation = 3; + + // The descriptive name for this cluster as it appears in UIs. + // Must be unique per zone. + string display_name = 4; + + // The number of serve nodes allocated to this cluster. + int32 serve_nodes = 5; + + // What storage type to use for tables in this cluster. Only configurable at + // cluster creation time. If unspecified, STORAGE_SSD will be used. + StorageType default_storage_type = 8; +} + +enum StorageType { + // The storage type used is unspecified. + STORAGE_UNSPECIFIED = 0; + + // Data will be stored in SSD, providing low and consistent latencies. + STORAGE_SSD = 1; +} diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_service.proto b/gcloud_bigtable/_generated/_bigtable_cluster_service.proto new file mode 100644 index 0000000..e65bca4 --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_cluster_service.proto @@ -0,0 +1,129 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + +import "google/api/annotations.proto"; +import "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto"; +import "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/empty.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterServicesProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// Service for managing zonal Cloud Bigtable resources. +service BigtableClusterService { + // Lists the supported zones for the given project. + rpc ListZones(ListZonesRequest) returns (ListZonesResponse) { + option (google.api.http) = { get: "/v1/{name=projects/*}/zones" }; + } + + // Gets information about a particular cluster. + rpc GetCluster(GetClusterRequest) returns (Cluster) { + option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*}" }; + } + + // Lists all clusters in the given project, along with any zones for which + // cluster information could not be retrieved. + rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) { + option (google.api.http) = { get: "/v1/{name=projects/*}/aggregated/clusters" }; + } + + // Creates a cluster and begins preparing it to begin serving. The returned + // cluster embeds as its "current_operation" a long-running operation which + // can be used to track the progress of turning up the new cluster. + // Immediately upon completion of this request: + // * The cluster will be readable via the API, with all requested attributes + // but no allocated resources. + // Until completion of the embedded operation: + // * Cancelling the operation will render the cluster immediately unreadable + // via the API. + // * All other attempts to modify or delete the cluster will be rejected. + // Upon completion of the embedded operation: + // * Billing for all successfully-allocated resources will begin (some types + // may have lower than the requested levels). + // * New tables can be created in the cluster. + // * The cluster's allocated resource levels will be readable via the API. + // The embedded operation's "metadata" field type is + // [CreateClusterMetadata][google.bigtable.admin.cluster.v1.CreateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + rpc CreateCluster(CreateClusterRequest) returns (Cluster) { + option (google.api.http) = { post: "/v1/{name=projects/*/zones/*}/clusters" body: "*" }; + } + + // Updates a cluster, and begins allocating or releasing resources as + // requested. The returned cluster embeds as its "current_operation" a + // long-running operation which can be used to track the progress of updating + // the cluster. + // Immediately upon completion of this request: + // * For resource types where a decrease in the cluster's allocation has been + // requested, billing will be based on the newly-requested level. + // Until completion of the embedded operation: + // * Cancelling the operation will set its metadata's "cancelled_at_time", + // and begin restoring resources to their pre-request values. The operation + // is guaranteed to succeed at undoing all resource changes, after which + // point it will terminate with a CANCELLED status. + // * All other attempts to modify or delete the cluster will be rejected. + // * Reading the cluster via the API will continue to give the pre-request + // resource levels. + // Upon completion of the embedded operation: + // * Billing will begin for all successfully-allocated resources (some types + // may have lower than the requested levels). + // * All newly-reserved resources will be available for serving the cluster's + // tables. + // * The cluster's new resource levels will be readable via the API. + // [UpdateClusterMetadata][google.bigtable.admin.cluster.v1.UpdateClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + rpc UpdateCluster(Cluster) returns (Cluster) { + option (google.api.http) = { put: "/v1/{name=projects/*/zones/*/clusters/*}" body: "*" }; + } + + // Marks a cluster and all of its tables for permanent deletion in 7 days. + // Immediately upon completion of the request: + // * Billing will cease for all of the cluster's reserved resources. + // * The cluster's "delete_time" field will be set 7 days in the future. + // Soon afterward: + // * All tables within the cluster will become unavailable. + // Prior to the cluster's "delete_time": + // * The cluster can be recovered with a call to UndeleteCluster. + // * All other attempts to modify or delete the cluster will be rejected. + // At the cluster's "delete_time": + // * The cluster and *all of its tables* will immediately and irrevocably + // disappear from the API, and their data will be permanently deleted. + rpc DeleteCluster(DeleteClusterRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*}" }; + } + + // Cancels the scheduled deletion of an cluster and begins preparing it to + // resume serving. The returned operation will also be embedded as the + // cluster's "current_operation". + // Immediately upon completion of this request: + // * The cluster's "delete_time" field will be unset, protecting it from + // automatic deletion. + // Until completion of the returned operation: + // * The operation cannot be cancelled. + // Upon completion of the returned operation: + // * Billing for the cluster's resources will resume. + // * All tables within the cluster will be available. + // [UndeleteClusterMetadata][google.bigtable.admin.cluster.v1.UndeleteClusterMetadata] The embedded operation's "response" field type is + // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. + rpc UndeleteCluster(UndeleteClusterRequest) returns (google.longrunning.Operation) { + option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*}:undelete" body: "null" }; + } +} diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto new file mode 100644 index 0000000..3291969 --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto @@ -0,0 +1,134 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.cluster.v1; + +import "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto"; +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableClusterServiceMessagesProto"; +option java_package = "com.google.bigtable.admin.cluster.v1"; + + +// Request message for BigtableClusterService.ListZones. +message ListZonesRequest { + // The unique name of the project for which a list of supported zones is + // requested. + // Values are of the form projects/ + string name = 1; +} + +// Response message for BigtableClusterService.ListZones. +message ListZonesResponse { + // The list of requested zones. + repeated Zone zones = 1; +} + +// Request message for BigtableClusterService.GetCluster. +message GetClusterRequest { + // The unique name of the requested cluster. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Request message for BigtableClusterService.ListClusters. +message ListClustersRequest { + // The unique name of the project for which a list of clusters is requested. + // Values are of the form projects/ + string name = 1; +} + +// Response message for BigtableClusterService.ListClusters. +message ListClustersResponse { + // The list of requested Clusters. + repeated Cluster clusters = 1; + + // The zones for which clusters could not be retrieved. + repeated Zone failed_zones = 2; +} + +// Request message for BigtableClusterService.CreateCluster. +message CreateClusterRequest { + // The unique name of the zone in which to create the cluster. + // Values are of the form projects//zones/ + string name = 1; + + // The id to be used when referring to the new cluster within its zone, + // e.g. just the "test-cluster" section of the full name + // "projects//zones//clusters/test-cluster". + string cluster_id = 2; + + // The cluster to create. + // The "name", "delete_time", and "current_operation" fields must be left + // blank. + Cluster cluster = 3; +} + +// Metadata type for the operation returned by +// BigtableClusterService.CreateCluster. +message CreateClusterMetadata { + // The request which prompted the creation of this operation. + CreateClusterRequest original_request = 1; + + // The time at which original_request was received. + google.protobuf.Timestamp request_time = 2; + + // The time at which this operation failed or was completed successfully. + google.protobuf.Timestamp finish_time = 3; +} + +// Metadata type for the operation returned by +// BigtableClusterService.UpdateCluster. +message UpdateClusterMetadata { + // The request which prompted the creation of this operation. + Cluster original_request = 1; + + // The time at which original_request was received. + google.protobuf.Timestamp request_time = 2; + + // The time at which this operation was cancelled. If set, this operation is + // in the process of undoing itself (which is guaranteed to succeed) and + // cannot be cancelled again. + google.protobuf.Timestamp cancel_time = 3; + + // The time at which this operation failed or was completed successfully. + google.protobuf.Timestamp finish_time = 4; +} + +// Request message for BigtableClusterService.DeleteCluster. +message DeleteClusterRequest { + // The unique name of the cluster to be deleted. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Request message for BigtableClusterService.UndeleteCluster. +message UndeleteClusterRequest { + // The unique name of the cluster to be un-deleted. + // Values are of the form projects//zones//clusters/ + string name = 1; +} + +// Metadata type for the operation returned by +// BigtableClusterService.UndeleteCluster. +message UndeleteClusterMetadata { + // The time at which the original request was received. + google.protobuf.Timestamp request_time = 1; + + // The time at which this operation failed or was completed successfully. + google.protobuf.Timestamp finish_time = 2; +} diff --git a/gcloud_bigtable/_generated/_bigtable_data.proto b/gcloud_bigtable/_generated/_bigtable_data.proto new file mode 100644 index 0000000..86234d2 --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_data.proto @@ -0,0 +1,500 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +option java_multiple_files = true; +option java_outer_classname = "BigtableDataProto"; +option java_package = "com.google.bigtable.v1"; + + +// Specifies the complete (requested) contents of a single row of a table. +// Rows which exceed 256MiB in size cannot be read in full. +message Row { + // The unique key which identifies this row within its table. This is the same + // key that's used to identify the row in, for example, a MutateRowRequest. + // May contain any non-empty byte string up to 4KiB in length. + bytes key = 1; + + // May be empty, but only if the entire row is empty. + // The mutual ordering of column families is not specified. + repeated Family families = 2; +} + +// Specifies (some of) the contents of a single row/column family of a table. +message Family { + // The unique key which identifies this family within its row. This is the + // same key that's used to identify the family in, for example, a RowFilter + // which sets its "family_name_regex_filter" field. + // Must match [-_.a-zA-Z0-9]+, except that AggregatingRowProcessors may + // produce cells in a sentinel family with an empty name. + // Must be no greater than 64 characters in length. + string name = 1; + + // Must not be empty. Sorted in order of increasing "qualifier". + repeated Column columns = 2; +} + +// Specifies (some of) the contents of a single row/column of a table. +message Column { + // The unique key which identifies this column within its family. This is the + // same key that's used to identify the column in, for example, a RowFilter + // which sets its "column_qualifier_regex_filter" field. + // May contain any byte string, including the empty string, up to 16kiB in + // length. + bytes qualifier = 1; + + // Must not be empty. Sorted in order of decreasing "timestamp_micros". + repeated Cell cells = 2; +} + +// Specifies (some of) the contents of a single row/column/timestamp of a table. +message Cell { + // The cell's stored timestamp, which also uniquely identifies it within + // its column. + // Values are always expressed in microseconds, but individual tables may set + // a coarser "granularity" to further restrict the allowed values. For + // example, a table which specifies millisecond granularity will only allow + // values of "timestamp_micros" which are multiples of 1000. + int64 timestamp_micros = 1; + + // The value stored in the cell. + // May contain any byte string, including the empty string, up to 100MiB in + // length. + bytes value = 2; + + // Labels applied to the cell by a [RowFilter][google.bigtable.v1.RowFilter]. + repeated string labels = 3; +} + +// Specifies a contiguous range of rows. +message RowRange { + // Inclusive lower bound. If left empty, interpreted as the empty string. + bytes start_key = 2; + + // Exclusive upper bound. If left empty, interpreted as infinity. + bytes end_key = 3; +} + +// Specifies a contiguous range of columns within a single column family. +// The range spans from : to +// :, where both bounds can be either inclusive or +// exclusive. +message ColumnRange { + // The name of the column family within which this range falls. + string family_name = 1; + + // The column qualifier at which to start the range (within 'column_family'). + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_qualifier { + // Used when giving an inclusive lower bound for the range. + bytes start_qualifier_inclusive = 2; + + // Used when giving an exclusive lower bound for the range. + bytes start_qualifier_exclusive = 3; + } + + // The column qualifier at which to end the range (within 'column_family'). + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_qualifier { + // Used when giving an inclusive upper bound for the range. + bytes end_qualifier_inclusive = 4; + + // Used when giving an exclusive upper bound for the range. + bytes end_qualifier_exclusive = 5; + } +} + +// Specified a contiguous range of microsecond timestamps. +message TimestampRange { + // Inclusive lower bound. If left empty, interpreted as 0. + int64 start_timestamp_micros = 1; + + // Exclusive upper bound. If left empty, interpreted as infinity. + int64 end_timestamp_micros = 2; +} + +// Specifies a contiguous range of raw byte values. +message ValueRange { + // The value at which to start the range. + // If neither field is set, interpreted as the empty string, inclusive. + oneof start_value { + // Used when giving an inclusive lower bound for the range. + bytes start_value_inclusive = 1; + + // Used when giving an exclusive lower bound for the range. + bytes start_value_exclusive = 2; + } + + // The value at which to end the range. + // If neither field is set, interpreted as the infinite string, exclusive. + oneof end_value { + // Used when giving an inclusive upper bound for the range. + bytes end_value_inclusive = 3; + + // Used when giving an exclusive upper bound for the range. + bytes end_value_exclusive = 4; + } +} + +// Takes a row as input and produces an alternate view of the row based on +// specified rules. For example, a RowFilter might trim down a row to include +// just the cells from columns matching a given regular expression, or might +// return all the cells of a row but not their values. More complicated filters +// can be composed out of these components to express requests such as, "within +// every column of a particular family, give just the two most recent cells +// which are older than timestamp X." +// +// There are two broad categories of RowFilters (true filters and transformers), +// as well as two ways to compose simple filters into more complex ones +// (chains and interleaves). They work as follows: +// +// * True filters alter the input row by excluding some of its cells wholesale +// from the output row. An example of a true filter is the "value_regex_filter", +// which excludes cells whose values don't match the specified pattern. All +// regex true filters use RE2 syntax (https://github.com/google/re2/wiki/Syntax) +// in raw byte mode (RE2::Latin1), and are evaluated as full matches. An +// important point to keep in mind is that RE2(.) is equivalent by default to +// RE2([^\n]), meaning that it does not match newlines. When attempting to match +// an arbitrary byte, you should therefore use the escape sequence '\C', which +// may need to be further escaped as '\\C' in your client language. +// +// * Transformers alter the input row by changing the values of some of its +// cells in the output, without excluding them completely. Currently, the only +// supported transformer is the "strip_value_transformer", which replaces every +// cell's value with the empty string. +// +// * Chains and interleaves are described in more detail in the +// RowFilter.Chain and RowFilter.Interleave documentation. +// +// The total serialized size of a RowFilter message must not +// exceed 4096 bytes, and RowFilters may not be nested within each other +// (in Chains or Interleaves) to a depth of more than 20. +message RowFilter { + // A RowFilter which sends rows through several RowFilters in sequence. + message Chain { + // The elements of "filters" are chained together to process the input row: + // in row -> f(0) -> intermediate row -> f(1) -> ... -> f(N) -> out row + // The full chain is executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which sends each row to each of several component + // RowFilters and interleaves the results. + message Interleave { + // The elements of "filters" all process a copy of the input row, and the + // results are pooled, sorted, and combined into a single output row. + // If multiple cells are produced with the same column and timestamp, + // they will all appear in the output row in an unspecified mutual order. + // Consider the following example, with three filters: + // + // input row + // | + // ----------------------------------------------------- + // | | | + // f(0) f(1) f(2) + // | | | + // 1: foo,bar,10,x foo,bar,10,z far,bar,7,a + // 2: foo,blah,11,z far,blah,5,x far,blah,5,x + // | | | + // ----------------------------------------------------- + // | + // 1: foo,bar,10,z // could have switched with #2 + // 2: foo,bar,10,x // could have switched with #1 + // 3: foo,blah,11,z + // 4: far,bar,7,a + // 5: far,blah,5,x // identical to #6 + // 6: far,blah,5,x // identical to #5 + // All interleaved filters are executed atomically. + repeated RowFilter filters = 1; + } + + // A RowFilter which evaluates one of two possible RowFilters, depending on + // whether or not a predicate RowFilter outputs any cells from the input row. + // + // IMPORTANT NOTE: The predicate filter does not execute atomically with the + // true and false filters, which may lead to inconsistent or unexpected + // results. Additionally, Condition filters have poor performance, especially + // when filters are set for the false condition. + message Condition { + // If "predicate_filter" outputs any cells, then "true_filter" will be + // evaluated on the input row. Otherwise, "false_filter" will be evaluated. + RowFilter predicate_filter = 1; + + // The filter to apply to the input row if "predicate_filter" returns any + // results. If not provided, no results will be returned in the true case. + RowFilter true_filter = 2; + + // The filter to apply to the input row if "predicate_filter" does not + // return any results. If not provided, no results will be returned in the + // false case. + RowFilter false_filter = 3; + } + + // Which of the possible RowFilter types to apply. If none are set, this + // RowFilter returns all cells in the input row. + oneof filter { + // Applies several RowFilters to the data in sequence, progressively + // narrowing the results. + Chain chain = 1; + + // Applies several RowFilters to the data in parallel and combines the + // results. + Interleave interleave = 2; + + // Applies one of two possible RowFilters to the data based on the output of + // a predicate RowFilter. + Condition condition = 3; + + // ADVANCED USE ONLY. + // Hook for introspection into the RowFilter. Outputs all cells directly to + // the output of the read rather than to any parent filter. Consider the + // following example: + // + // Chain( + // FamilyRegex("A"), + // Interleave( + // All(), + // Chain(Label("foo"), Sink()) + // ), + // QualifierRegex("B") + // ) + // + // A,A,1,w + // A,B,2,x + // B,B,4,z + // | + // FamilyRegex("A") + // | + // A,A,1,w + // A,B,2,x + // | + // +------------+-------------+ + // | | + // All() Label(foo) + // | | + // A,A,1,w A,A,1,w,labels:[foo] + // A,B,2,x A,B,2,x,labels:[foo] + // | | + // | Sink() --------------+ + // | | | + // +------------+ x------+ A,A,1,w,labels:[foo] + // | A,B,2,x,labels:[foo] + // A,A,1,w | + // A,B,2,x | + // | | + // QualifierRegex("B") | + // | | + // A,B,2,x | + // | | + // +--------------------------------+ + // | + // A,A,1,w,labels:[foo] + // A,B,2,x,labels:[foo] // could be switched + // A,B,2,x // could be switched + // + // Despite being excluded by the qualifier filter, a copy of every cell + // that reaches the sink is present in the final result. + // + // As with an [Interleave][google.bigtable.v1.RowFilter.Interleave], + // duplicate cells are possible, and appear in an unspecified mutual order. + // In this case we have a duplicate with column "A:B" and timestamp 2, + // because one copy passed through the all filter while the other was + // passed through the label and sink. Note that one copy has label "foo", + // while the other does not. + // + // Cannot be used within the `predicate_filter`, `true_filter`, or + // `false_filter` of a [Condition][google.bigtable.v1.RowFilter.Condition]. + bool sink = 16; + + // Matches all cells, regardless of input. Functionally equivalent to + // leaving `filter` unset, but included for completeness. + bool pass_all_filter = 17; + + // Does not match any cells, regardless of input. Useful for temporarily + // disabling just part of a filter. + bool block_all_filter = 18; + + // Matches only cells from rows whose keys satisfy the given RE2 regex. In + // other words, passes through the entire row when the key matches, and + // otherwise produces an empty row. + // Note that, since row keys can contain arbitrary bytes, the '\C' escape + // sequence must be used if a true wildcard is desired. The '.' character + // will not match the new line character '\n', which may be present in a + // binary key. + bytes row_key_regex_filter = 4; + + // Matches all cells from a row with probability p, and matches no cells + // from the row with probability 1-p. + double row_sample_filter = 14; + + // Matches only cells from columns whose families satisfy the given RE2 + // regex. For technical reasons, the regex must not contain the ':' + // character, even if it is not being used as a literal. + // Note that, since column families cannot contain the new line character + // '\n', it is sufficient to use '.' as a full wildcard when matching + // column family names. + string family_name_regex_filter = 5; + + // Matches only cells from columns whose qualifiers satisfy the given RE2 + // regex. + // Note that, since column qualifiers can contain arbitrary bytes, the '\C' + // escape sequence must be used if a true wildcard is desired. The '.' + // character will not match the new line character '\n', which may be + // present in a binary qualifier. + bytes column_qualifier_regex_filter = 6; + + // Matches only cells from columns within the given range. + ColumnRange column_range_filter = 7; + + // Matches only cells with timestamps within the given range. + TimestampRange timestamp_range_filter = 8; + + // Matches only cells with values that satisfy the given regular expression. + // Note that, since cell values can contain arbitrary bytes, the '\C' escape + // sequence must be used if a true wildcard is desired. The '.' character + // will not match the new line character '\n', which may be present in a + // binary value. + bytes value_regex_filter = 9; + + // Matches only cells with values that fall within the given range. + ValueRange value_range_filter = 15; + + // Skips the first N cells of each row, matching all subsequent cells. + int32 cells_per_row_offset_filter = 10; + + // Matches only the first N cells of each row. + int32 cells_per_row_limit_filter = 11; + + // Matches only the most recent N cells within each column. For example, + // if N=2, this filter would match column "foo:bar" at timestamps 10 and 9, + // skip all earlier cells in "foo:bar", and then begin matching again in + // column "foo:bar2". + int32 cells_per_column_limit_filter = 12; + + // Replaces each cell's value with the empty string. + bool strip_value_transformer = 13; + + // Applies the given label to all cells in the output row. This allows + // the client to determine which results were produced from which part of + // the filter. + // + // Values must be at most 15 characters in length, and match the RE2 + // pattern [a-z0-9\\-]+ + // + // Due to a technical limitation, it is not currently possible to apply + // multiple labels to a cell. As a result, a Chain may have no more than + // one sub-filter which contains a apply_label_transformer. It is okay for + // an Interleave to contain multiple apply_label_transformers, as they will + // be applied to separate copies of the input. This may be relaxed in the + // future. + string apply_label_transformer = 19; + } +} + +// Specifies a particular change to be made to the contents of a row. +message Mutation { + // A Mutation which sets the value of the specified cell. + message SetCell { + // The name of the family into which new data should be written. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column into which new data should be written. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The timestamp of the cell into which new data should be written. + // Use -1 for current Bigtable server time. + // Otherwise, the client should set this value itself, noting that the + // default value is a timestamp of zero if the field is left unspecified. + // Values must match the "granularity" of the table (e.g. micros, millis). + int64 timestamp_micros = 3; + + // The value to be written into the specified cell. + bytes value = 4; + } + + // A Mutation which deletes cells from the specified column, optionally + // restricting the deletions to a given timestamp range. + message DeleteFromColumn { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column from which cells should be deleted. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The range of timestamps within which cells should be deleted. + TimestampRange time_range = 3; + } + + // A Mutation which deletes all cells from the specified column family. + message DeleteFromFamily { + // The name of the family from which cells should be deleted. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + } + + // A Mutation which deletes all cells from the containing row. + message DeleteFromRow { + + } + + // Which of the possible Mutation types to apply. + oneof mutation { + // Set a cell's value. + SetCell set_cell = 1; + + // Deletes cells from a column. + DeleteFromColumn delete_from_column = 2; + + // Deletes cells from a column family. + DeleteFromFamily delete_from_family = 3; + + // Deletes cells from the entire row. + DeleteFromRow delete_from_row = 4; + } +} + +// Specifies an atomic read/modify/write operation on the latest value of the +// specified column. +message ReadModifyWriteRule { + // The name of the family to which the read/modify/write should be applied. + // Must match [-_.a-zA-Z0-9]+ + string family_name = 1; + + // The qualifier of the column to which the read/modify/write should be + // applied. + // Can be any byte string, including the empty string. + bytes column_qualifier = 2; + + // The rule used to determine the column's new latest value from its current + // latest value. + oneof rule { + // Rule specifying that "append_value" be appended to the existing value. + // If the targeted cell is unset, it will be treated as containing the + // empty string. + bytes append_value = 3; + + // Rule specifying that "increment_amount" be added to the existing value. + // If the targeted cell is unset, it will be treated as containing a zero. + // Otherwise, the targeted cell must contain an 8-byte value (interpreted + // as a 64-bit big-endian signed integer), or the entire request will fail. + int64 increment_amount = 4; + } +} diff --git a/gcloud_bigtable/_generated/_bigtable_service.proto b/gcloud_bigtable/_generated/_bigtable_service.proto new file mode 100644 index 0000000..7f9109f --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_service.proto @@ -0,0 +1,66 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +import "google/api/annotations.proto"; +import "google/bigtable/v1/bigtable_data.proto"; +import "google/bigtable/v1/bigtable_service_messages.proto"; +import "google/protobuf/empty.proto"; + +option java_generic_services = true; +option java_multiple_files = true; +option java_outer_classname = "BigtableServicesProto"; +option java_package = "com.google.bigtable.v1"; + + +// Service for reading from and writing to existing Bigtables. +service BigtableService { + // Streams back the contents of all requested rows, optionally applying + // the same Reader filter to each. Depending on their size, rows may be + // broken up across multiple responses, but atomicity of each row will still + // be preserved. + rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { + option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read" body: "*" }; + } + + // Returns a sample of row keys in the table. The returned row keys will + // delimit contiguous sections of the table of approximately equal size, + // which can be used to break up the data for distributed tasks like + // mapreduces. + rpc SampleRowKeys(SampleRowKeysRequest) returns (stream SampleRowKeysResponse) { + option (google.api.http) = { get: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys" }; + } + + // Mutates a row atomically. Cells already present in the row are left + // unchanged unless explicitly changed by 'mutation'. + rpc MutateRow(MutateRowRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate" body: "*" }; + } + + // Mutates a row atomically based on the output of a predicate Reader filter. + rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) { + option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate" body: "*" }; + } + + // Modifies a row atomically, reading the latest existing timestamp/value from + // the specified columns and writing a new value at + // max(existing timestamp, current server time) based on pre-defined + // read/modify/write rules. Returns the new contents of all modified cells. + rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (Row) { + option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite" body: "*" }; + } +} diff --git a/gcloud_bigtable/_generated/_bigtable_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_service_messages.proto new file mode 100644 index 0000000..dacc0fc --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_service_messages.proto @@ -0,0 +1,177 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.v1; + +import "google/bigtable/v1/bigtable_data.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableServiceMessagesProto"; +option java_package = "com.google.bigtable.v1"; + + +// Request message for BigtableServer.ReadRows. +message ReadRowsRequest { + // The unique name of the table from which to read. + string table_name = 1; + + // If neither row_key nor row_range is set, reads from all rows. + oneof target { + // The key of a single row from which to read. + bytes row_key = 2; + + // A range of rows from which to read. + RowRange row_range = 3; + } + + // The filter to apply to the contents of the specified row(s). If unset, + // reads the entire table. + RowFilter filter = 5; + + // By default, rows are read sequentially, producing results which are + // guaranteed to arrive in increasing row order. Setting + // "allow_row_interleaving" to true allows multiple rows to be interleaved in + // the response stream, which increases throughput but breaks this guarantee, + // and may force the client to use more memory to buffer partially-received + // rows. Cannot be set to true when specifying "num_rows_limit". + bool allow_row_interleaving = 6; + + // The read will terminate after committing to N rows' worth of results. The + // default (zero) is to return all results. + // Note that "allow_row_interleaving" cannot be set to true when this is set. + int64 num_rows_limit = 7; +} + +// Response message for BigtableService.ReadRows. +message ReadRowsResponse { + // Specifies a piece of a row's contents returned as part of the read + // response stream. + message Chunk { + oneof chunk { + // A subset of the data from a particular row. As long as no "reset_row" + // is received in between, multiple "row_contents" from the same row are + // from the same atomic view of that row, and will be received in the + // expected family/column/timestamp order. + Family row_contents = 1; + + // Indicates that the client should drop all previous chunks for + // "row_key", as it will be re-read from the beginning. + bool reset_row = 2; + + // Indicates that the client can safely process all previous chunks for + // "row_key", as its data has been fully read. + bool commit_row = 3; + } + } + + // The key of the row for which we're receiving data. + // Results will be received in increasing row key order, unless + // "allow_row_interleaving" was specified in the request. + bytes row_key = 1; + + // One or more chunks of the row specified by "row_key". + repeated Chunk chunks = 2; +} + +// Request message for BigtableService.SampleRowKeys. +message SampleRowKeysRequest { + // The unique name of the table from which to sample row keys. + string table_name = 1; +} + +// Response message for BigtableService.SampleRowKeys. +message SampleRowKeysResponse { + // Sorted streamed sequence of sample row keys in the table. The table might + // have contents before the first row key in the list and after the last one, + // but a key containing the empty string indicates "end of table" and will be + // the last response given, if present. + // Note that row keys in this list may not have ever been written to or read + // from, and users should therefore not make any assumptions about the row key + // structure that are specific to their use case. + bytes row_key = 1; + + // Approximate total storage space used by all rows in the table which precede + // "row_key". Buffering the contents of all rows between two subsequent + // samples would require space roughly equal to the difference in their + // "offset_bytes" fields. + int64 offset_bytes = 2; +} + +// Request message for BigtableService.MutateRow. +message MutateRowRequest { + // The unique name of the table to which the mutation should be applied. + string table_name = 1; + + // The key of the row to which the mutation should be applied. + bytes row_key = 2; + + // Changes to be atomically applied to the specified row. Entries are applied + // in order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry and at most 100000. + repeated Mutation mutations = 3; +} + +// Request message for BigtableService.CheckAndMutateRowRequest +message CheckAndMutateRowRequest { + // The unique name of the table to which the conditional mutation should be + // applied. + string table_name = 1; + + // The key of the row to which the conditional mutation should be applied. + bytes row_key = 2; + + // The filter to be applied to the contents of the specified row. Depending + // on whether or not any results are yielded, either "true_mutations" or + // "false_mutations" will be executed. If unset, checks that the row contains + // any values at all. + RowFilter predicate_filter = 6; + + // Changes to be atomically applied to the specified row if "predicate_filter" + // yields at least one cell when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "false_mutations" is empty, and at most + // 100000. + repeated Mutation true_mutations = 4; + + // Changes to be atomically applied to the specified row if "predicate_filter" + // does not yield any cells when applied to "row_key". Entries are applied in + // order, meaning that earlier mutations can be masked by later ones. + // Must contain at least one entry if "true_mutations" is empty, and at most + // 100000. + repeated Mutation false_mutations = 5; +} + +// Response message for BigtableService.CheckAndMutateRowRequest. +message CheckAndMutateRowResponse { + // Whether or not the request's "predicate_filter" yielded any results for + // the specified row. + bool predicate_matched = 1; +} + +// Request message for BigtableService.ReadModifyWriteRowRequest. +message ReadModifyWriteRowRequest { + // The unique name of the table to which the read/modify/write rules should be + // applied. + string table_name = 1; + + // The key of the row to which the read/modify/write rules should be applied. + bytes row_key = 2; + + // Rules specifying how the specified row's contents are to be transformed + // into writes. Entries are applied in order, meaning that earlier rules will + // affect the results of later ones. + repeated ReadModifyWriteRule rules = 3; +} diff --git a/gcloud_bigtable/_generated/_bigtable_table_data.proto b/gcloud_bigtable/_generated/_bigtable_table_data.proto new file mode 100644 index 0000000..f81c878 --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_table_data.proto @@ -0,0 +1,125 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google/longrunning/operations.proto"; +import "google/protobuf/duration.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableDataProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +// A collection of user data indexed by row, column, and timestamp. +// Each table is served using the resources of its parent cluster. +message Table { + enum TimestampGranularity { + MILLIS = 0; + } + + // A unique identifier of the form + // /tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]* + string name = 1; + + // If this Table is in the process of being created, the Operation used to + // track its progress. As long as this operation is present, the Table will + // not accept any Table Admin or Read/Write requests. + google.longrunning.Operation current_operation = 2; + + // The column families configured for this table, mapped by column family id. + map column_families = 3; + + // The granularity (e.g. MILLIS, MICROS) at which timestamps are stored in + // this table. Timestamps not matching the granularity will be rejected. + // Cannot be changed once the table is created. + TimestampGranularity granularity = 4; +} + +// A set of columns within a table which share a common configuration. +message ColumnFamily { + // A unique identifier of the form /columnFamilies/[-_.a-zA-Z0-9]+ + // The last segment is the same as the "name" field in + // google.bigtable.v1.Family. + string name = 1; + + // Garbage collection expression specified by the following grammar: + // GC = EXPR + // | "" ; + // EXPR = EXPR, "||", EXPR (* lowest precedence *) + // | EXPR, "&&", EXPR + // | "(", EXPR, ")" (* highest precedence *) + // | PROP ; + // PROP = "version() >", NUM32 + // | "age() >", NUM64, [ UNIT ] ; + // NUM32 = non-zero-digit { digit } ; (* # NUM32 <= 2^32 - 1 *) + // NUM64 = non-zero-digit { digit } ; (* # NUM64 <= 2^63 - 1 *) + // UNIT = "d" | "h" | "m" (* d=days, h=hours, m=minutes, else micros *) + // GC expressions can be up to 500 characters in length + // + // The different types of PROP are defined as follows: + // version() - cell index, counting from most recent and starting at 1 + // age() - age of the cell (current time minus cell timestamp) + // + // Example: "version() > 3 || (age() > 3d && version() > 1)" + // drop cells beyond the most recent three, and drop cells older than three + // days unless they're the most recent cell in the row/column + // + // Garbage collection executes opportunistically in the background, and so + // it's possible for reads to return a cell even if it matches the active GC + // expression for its family. + string gc_expression = 2; + + // Garbage collection rule specified as a protobuf. + // Supersedes `gc_expression`. + // Must serialize to at most 500 bytes. + // + // NOTE: Garbage collection executes opportunistically in the background, and + // so it's possible for reads to return a cell even if it matches the active + // GC expression for its family. + GcRule gc_rule = 3; +} + +// Rule for determining which cells to delete during garbage collection. +message GcRule { + // A GcRule which deletes cells matching all of the given rules. + message Intersection { + // Only delete cells which would be deleted by every element of `rules`. + repeated GcRule rules = 1; + } + + // A GcRule which deletes cells matching any of the given rules. + message Union { + // Delete cells which would be deleted by any element of `rules`. + repeated GcRule rules = 1; + } + + oneof rule { + // Delete all cells in a column except the most recent N. + int32 max_num_versions = 1; + + // Delete cells in a column older than the given age. + // Values must be at least one millisecond, and will be truncated to + // microsecond granularity. + google.protobuf.Duration max_age = 2; + + // Delete cells that would be deleted by every nested rule. + Intersection intersection = 3; + + // Delete cells that would be deleted by any nested rule. + Union union = 4; + } +} diff --git a/gcloud_bigtable/_generated/_bigtable_table_service.proto b/gcloud_bigtable/_generated/_bigtable_table_service.proto new file mode 100644 index 0000000..417409c --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_table_service.proto @@ -0,0 +1,74 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google/api/annotations.proto"; +import "google/bigtable/admin/table/v1/bigtable_table_data.proto"; +import "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto"; +import "google/protobuf/empty.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableServicesProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +// Service for creating, configuring, and deleting Cloud Bigtable tables. +// Provides access to the table schemas only, not the data stored within the tables. +service BigtableTableService { + // Creates a new table, to be served from a specified cluster. + // The table can be created with a full set of initial column families, + // specified in the request. + rpc CreateTable(CreateTableRequest) returns (Table) { + option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*}/tables" body: "*" }; + } + + // Lists the names of all tables served from a specified cluster. + rpc ListTables(ListTablesRequest) returns (ListTablesResponse) { + option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*}/tables" }; + } + + // Gets the schema of the specified table, including its column families. + rpc GetTable(GetTableRequest) returns (Table) { + option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}" }; + } + + // Permanently deletes a specified table and all of its data. + rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}" }; + } + + // Changes the name of a specified table. + // Cannot be used to move tables between clusters, zones, or projects. + rpc RenameTable(RenameTableRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename" body: "*" }; + } + + // Creates a new column family within a specified table. + rpc CreateColumnFamily(CreateColumnFamilyRequest) returns (ColumnFamily) { + option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies" body: "*" }; + } + + // Changes the configuration of a specified column family. + rpc UpdateColumnFamily(ColumnFamily) returns (ColumnFamily) { + option (google.api.http) = { put: "/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}" body: "*" }; + } + + // Permanently deletes a specified column family and all of its data. + rpc DeleteColumnFamily(DeleteColumnFamilyRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}" }; + } +} diff --git a/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto new file mode 100644 index 0000000..73f2a8c --- /dev/null +++ b/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto @@ -0,0 +1,101 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.bigtable.admin.table.v1; + +import "google/bigtable/admin/table/v1/bigtable_table_data.proto"; + +option java_multiple_files = true; +option java_outer_classname = "BigtableTableServiceMessagesProto"; +option java_package = "com.google.bigtable.admin.table.v1"; + + +message CreateTableRequest { + // The unique name of the cluster in which to create the new table. + string name = 1; + + // The name by which the new table should be referred to within the cluster, + // e.g. "foobar" rather than "/tables/foobar". + string table_id = 2; + + // The Table to create. The `name` field of the Table and all of its + // ColumnFamilies must be left blank, and will be populated in the response. + Table table = 3; + + // The optional list of row keys that will be used to initially split the + // table into several tablets (Tablets are similar to HBase regions). + // Given two split keys, "s1" and "s2", three tablets will be created, + // spanning the key ranges: [, s1), [s1, s2), [s2, ). + // + // Example: + // * Row keys := ["a", "apple", "custom", "customer_1", "customer_2", + // "other", "zz"] + // * initial_split_keys := ["apple", "customer_1", "customer_2", "other"] + // * Key assignment: + // - Tablet 1 [, apple) => {"a"}. + // - Tablet 2 [apple, customer_1) => {"apple", "custom"}. + // - Tablet 3 [customer_1, customer_2) => {"customer_1"}. + // - Tablet 4 [customer_2, other) => {"customer_2"}. + // - Tablet 5 [other, ) => {"other", "zz"}. + repeated string initial_split_keys = 4; +} + +message ListTablesRequest { + // The unique name of the cluster for which tables should be listed. + string name = 1; +} + +message ListTablesResponse { + // The tables present in the requested cluster. + // At present, only the names of the tables are populated. + repeated Table tables = 1; +} + +message GetTableRequest { + // The unique name of the requested table. + string name = 1; +} + +message DeleteTableRequest { + // The unique name of the table to be deleted. + string name = 1; +} + +message RenameTableRequest { + // The current unique name of the table. + string name = 1; + + // The new name by which the table should be referred to within its containing + // cluster, e.g. "foobar" rather than "/tables/foobar". + string new_id = 2; +} + +message CreateColumnFamilyRequest { + // The unique name of the table in which to create the new column family. + string name = 1; + + // The name by which the new column family should be referred to within the + // table, e.g. "foobar" rather than "/columnFamilies/foobar". + string column_family_id = 2; + + // The column family to create. The `name` field must be left blank. + ColumnFamily column_family = 3; +} + +message DeleteColumnFamilyRequest { + // The unique name of the column family to be deleted. + string name = 1; +} diff --git a/gcloud_bigtable/_generated/_duration.proto b/gcloud_bigtable/_generated/_duration.proto new file mode 100644 index 0000000..15e9d44 --- /dev/null +++ b/gcloud_bigtable/_generated/_duration.proto @@ -0,0 +1,78 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "DurationProto"; +option java_package = "com.google.protobuf"; + + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (durations.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/gcloud_bigtable/_generated/_empty.proto b/gcloud_bigtable/_generated/_empty.proto new file mode 100644 index 0000000..43b06e8 --- /dev/null +++ b/gcloud_bigtable/_generated/_empty.proto @@ -0,0 +1,34 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_multiple_files = true; +option java_outer_classname = "EmptyProto"; +option java_package = "com.google.protobuf"; + + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +message Empty { + +} diff --git a/gcloud_bigtable/_generated/_http.proto b/gcloud_bigtable/_generated/_http.proto new file mode 100644 index 0000000..ce07aa1 --- /dev/null +++ b/gcloud_bigtable/_generated/_http.proto @@ -0,0 +1,127 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; + + +// `HttpRule` defines the mapping of an RPC method to one or more HTTP REST API +// methods. The mapping determines what portions of the request message are +// populated from the path, query parameters, or body of the HTTP request. The +// mapping is typically specified as an `google.api.http` annotation, see +// "google/api/annotations.proto" for details. +// +// The mapping consists of a mandatory field specifying a path template and an +// optional `body` field specifying what data is represented in the HTTP request +// body. The field name for the path indicates the HTTP method. Example: +// +// ``` +// package google.storage.v2; +// +// import "google/api/annotations.proto"; +// +// service Storage { +// rpc CreateObject(CreateObjectRequest) returns (Object) { +// option (google.api.http) { +// post: "/v2/{bucket_name=buckets/*}/objects" +// body: "object" +// }; +// }; +// } +// ``` +// +// Here `bucket_name` and `object` bind to fields of the request message +// `CreateObjectRequest`. +// +// The rules for mapping HTTP path, query parameters, and body fields +// to the request message are as follows: +// +// 1. The `body` field specifies either `*` or a field path, or is +// omitted. If omitted, it assumes there is no HTTP body. +// 2. Leaf fields (recursive expansion of nested messages in the +// request) can be classified into three types: +// (a) Matched in the URL template. +// (b) Covered by body (if body is `*`, everything except (a) fields; +// else everything under the body field) +// (c) All other fields. +// 3. URL query parameters found in the HTTP request are mapped to (c) fields. +// 4. Any body sent with an HTTP request can contain only (b) fields. +// +// The syntax of the path template is as follows: +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// `*` matches a single path component, `**` zero or more path components, and +// `LITERAL` a constant. A `Variable` can match an entire path as specified +// again by a template; this nested template must not contain further variables. +// If no template is given with a variable, it matches a single path component. +// The notation `{var}` is henceforth equivalent to `{var=*}`. +// +// Use CustomHttpPattern to specify any HTTP method that is not included in the +// pattern field, such as HEAD, or "*" to leave the HTTP method unspecified for +// a given URL path rule. The wild-card rule is useful for services that provide +// content to Web (HTML) clients. +message HttpRule { + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Used for listing and getting information about resources. + string get = 2; + + // Used for updating a resource. + string put = 3; + + // Used for creating a resource. + string post = 4; + + // Used for deleting a resource. + string delete = 5; + + // Used for updating a resource. + string patch = 6; + + // Custom pattern is used for defining custom verbs. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP body, or + // `*` for mapping all fields not captured by the path pattern to the HTTP + // body. + string body = 7; + + // Additional HTTP bindings for the selector. Nested bindings must not + // specify a selector and must not contain additional bindings. + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/gcloud_bigtable/_generated/_operations.proto b/gcloud_bigtable/_generated/_operations.proto new file mode 100644 index 0000000..a358d0a --- /dev/null +++ b/gcloud_bigtable/_generated/_operations.proto @@ -0,0 +1,144 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.longrunning; + +import "google/api/annotations.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/empty.proto"; +import "google/rpc/status.proto"; + +option java_multiple_files = true; +option java_outer_classname = "OperationsProto"; +option java_package = "com.google.longrunning"; + + +// Manages long-running operations with an API service. +// +// When an API method normally takes long time to complete, it can be designed +// to return [Operation][google.longrunning.Operation] to the client, and the client can use this +// interface to receive the real response asynchronously by polling the +// operation resource, or using `google.watcher.v1.Watcher` interface to watch +// the response, or pass the operation resource to another API (such as Google +// Cloud Pub/Sub API) to receive the response. Any API service that returns +// long-running operations should implement the `Operations` interface so +// developers can have a consistent client experience. +service Operations { + // Gets the latest state of a long-running operation. Clients may use this + // method to poll the operation result at intervals as recommended by the API + // service. + rpc GetOperation(GetOperationRequest) returns (Operation) { + option (google.api.http) = { get: "/v1/{name=operations/**}" }; + } + + // Lists operations that match the specified filter in the request. If the + // server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { + option (google.api.http) = { get: "/v1/{name=operations}" }; + } + + // Starts asynchronous cancellation on a long-running operation. The server + // makes a best effort to cancel the operation, but success is not + // guaranteed. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. Clients may use + // [Operations.GetOperation] or other methods to check whether the + // cancellation succeeded or the operation completed despite cancellation. + rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{name=operations/**}:cancel" body: "*" }; + } + + // Deletes a long-running operation. It indicates the client is no longer + // interested in the operation result. It does not cancel the operation. + rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { delete: "/v1/{name=operations/**}" }; + } +} + +// This resource represents a long-running operation that is the result of a +// network API call. +message Operation { + // The name of the operation resource, which is only unique within the same + // service that originally returns it. + string name = 1; + + // Some service-specific metadata associated with the operation. It typically + // contains progress information and common metadata such as create time. + // Some services may not provide such metadata. Any method that returns a + // long-running operation should document the metadata type, if any. + google.protobuf.Any metadata = 2; + + // If the value is false, it means the operation is still in progress. + // If true, the operation is completed and the `result` is available. + bool done = 3; + + oneof result { + // The error result of the operation in case of failure. + google.rpc.Status error = 4; + + // The normal response of the operation in case of success. If the original + // method returns no data on success, such as `Delete`, the response will be + // `google.protobuf.Empty`. If the original method is standard + // `Get`/`Create`/`Update`, the response should be the resource. For other + // methods, the response should have the type `XxxResponse`, where `Xxx` + // is the original method name. For example, if the original method name + // is `TakeSnapshot()`, the inferred response type will be + // `TakeSnapshotResponse`. + google.protobuf.Any response = 5; + } +} + +// The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. +message GetOperationRequest { + // The name of the operation resource. + string name = 1; +} + +// The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsRequest { + // The name of the operation collection. + string name = 4; + + // The standard List filter. + string filter = 1; + + // The standard List page size. + int32 page_size = 2; + + // The standard List page token. + string page_token = 3; +} + +// The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsResponse { + // A list of operations that match the specified filter in the request. + repeated Operation operations = 1; + + // The standard List next-page token. + string next_page_token = 2; +} + +// The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. +message CancelOperationRequest { + // The name of the operation resource to be cancelled. + string name = 1; +} + +// The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. +message DeleteOperationRequest { + // The name of the operation resource to be deleted. + string name = 1; +} diff --git a/gcloud_bigtable/_generated/_status.proto b/gcloud_bigtable/_generated/_status.proto new file mode 100644 index 0000000..be97fa4 --- /dev/null +++ b/gcloud_bigtable/_generated/_status.proto @@ -0,0 +1,88 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.rpc; + +import "google/protobuf/any.proto"; + +option java_multiple_files = true; +option java_outer_classname = "StatusProto"; +option java_package = "com.google.rpc"; + + +// The `Status` defines a logical error model that is suitable for different +// programming environments, including REST APIs and RPC APIs. It is used by +// [gRPC](https://github.com/grpc). The error model is designed to be: +// +// - Simple to use and understand for most users. +// - Flexible enough to meet unexpected needs. +// +// # Overview +// The `Status` message contains 3 pieces of data: error code, error message, +// and error details. The error code should be an enum value of +// [google.rpc.Code][google.rpc.Code], but it may accept additional error codes if needed. The +// error message should be a developer-facing English message that helps +// developers *understand* and *resolve* the error. If a localized user-facing +// error message is needed, it can be sent in the error details or localized by +// the client. The optional error details may contain arbitrary information +// about the error. There is a predefined set of error detail types in the +// package `google.rpc` which can be used for common error conditions. +// +// # Language mapping +// The `Status` message is the logical representation of the error model, but it +// is not necessarily the actual wire format. When the `Status` message is +// exposed in different client libraries and different wire protocols, it can be +// mapped differently. For example, it will likely be mapped to some exceptions +// in Java, but more likely mapped to some error codes in C. +// +// # Other uses +// The error model and the `Status` message can be used in a variety of +// environments - either with or without APIs - to provide consistent developer +// experience across different environments. +// +// Example uses of this error model include: +// +// - Partial errors. If a service needs to return partial errors to the client, +// it may embed the `Status` in the normal response to indicate the partial +// errors. +// +// - Workflow errors. A typical workflow has multiple steps. Each step may +// have a `Status` message for error reporting purpose. +// +// - Batch operations. If a client uses batch request and batch response, the +// `Status` message should be used directly inside batch response, one for +// each error sub-response. +// +// - Asynchronous operations. If an API call embeds asynchronous operation +// results in its response, the status of those operations should be +// represented directly using the `Status` message. +// +// - Logging. If some API errors are stored in logs, the message `Status` could +// be used directly after any stripping needed for security/privacy reasons. +// +message Status { + // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. + int32 code = 1; + + // A developer-facing error message, which should be in English. Any + // user-facing error message should be localized and sent in the + // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. + string message = 2; + + // A list of messages that carry the error details. There will be a + // common set of message types for APIs to use. + repeated google.protobuf.Any details = 3; +} diff --git a/gcloud_bigtable/_generated/_timestamp.proto b/gcloud_bigtable/_generated/_timestamp.proto new file mode 100644 index 0000000..b135854 --- /dev/null +++ b/gcloud_bigtable/_generated/_timestamp.proto @@ -0,0 +1,89 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.protobuf; + +option java_generate_equals_and_hash = true; +option java_multiple_files = true; +option java_outer_classname = "TimestampProto"; +option java_package = "com.google.protobuf"; + + +// A Timestamp represents a point in time independent of any time zone +// or calendar, represented as seconds and fractions of seconds at +// nanosecond resolution in UTC Epoch time. It is encoded using the +// Proleptic Gregorian Calendar which extends the Gregorian calendar +// backwards to year one. It is encoded assuming all minutes are 60 +// seconds long, i.e. leap seconds are "smeared" so that no leap second +// table is needed for interpretation. Range is from +// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. +// By restricting to that range, we ensure that we can convert to +// and from RFC 3339 date strings. +// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// Example 5: Compute Timestamp from Python `datetime.datetime`. +// +// now = datetime.datetime.utcnow() +// seconds = int(time.mktime(now.timetuple())) +// nanos = now.microsecond * 1000 +// timestamp = Timestamp(seconds=seconds, nanos=nanos) +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/scripts/rewrite_imports.py b/scripts/rewrite_imports.py index 7685411..d985dac 100644 --- a/scripts/rewrite_imports.py +++ b/scripts/rewrite_imports.py @@ -49,8 +49,10 @@ def transform_old_to_new(line, old_module, new_module, "from {old_module} import ..." then checks if the line contains "import {old_module} ..." - and finally checks if the line starts with (ignoring whitespace) + then checks if the line starts with (ignoring whitespace) "{old_module} ..." + and finally checks if the line contians + "'some-dict-key': {old_module} ..." In any of these cases, "{old_module}" is replaced with "{new_module}". If none match, nothing is returned. @@ -96,6 +98,11 @@ def transform_old_to_new(line, old_module, new_module, # Only replace the first instance of the old_module. return line.replace(old_module, new_module, 1) + # Finally check for usage in dictionaries. + if ': ' + old_module in line: + # Only replace the first instance of the old_module. + return line.replace(': ' + old_module, ': ' + new_module, 1) + def transform_line(line): """Transforms an import line in a PB2 module. From 70be122b2275002d6e3ef4aabc4b2aa50062abba Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 15:43:30 -0800 Subject: [PATCH 29/65] Adding system tests for Table.rename and ColumnFamily.update. This was enabled in gRPC beta since we can now get information from errors. This also revealed that ColumnFamily.update was now implemented. --- gcloud_bigtable/column_family.py | 8 ++---- system_tests/run.py | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index aab9c82..1c5bb60 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -264,12 +264,8 @@ def update(self, timeout_seconds=None): .. note:: - The Bigtable Table Admin API currently returns - - ``BigtableTableService.UpdateColumnFamily is not yet implemented`` - - when this method is used. It's unclear when this method will - actually be supported by the API. + Only the GC rule can be updated. By changing the column family ID, + you will simply be referring to a different column family. :type timeout_seconds: int :param timeout_seconds: Number of seconds for request time-out. diff --git a/system_tests/run.py b/system_tests/run.py index d8b3978..f90253d 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -203,6 +203,26 @@ def test_create_table(self): sorted_tables = sorted(tables, key=name_attr) self.assertEqual(sorted_tables, expected_tables) + def test_rename_table(self): + from grpc.beta.interfaces import StatusCode + from grpc.framework.interfaces.face.face import LocalError + + temp_table_id = 'foo-bar-baz-table' + temp_table = CLUSTER.table(temp_table_id) + temp_table.create() + self.tables_to_delete.append(temp_table) + + exc_caught = None + try: + temp_table.rename(temp_table_id + '-alt') + except LocalError as exc: + exc_caught = exc # Python 3 syntax issue. + self.assertNotEqual(exc_caught, None) + self.assertEqual(exc_caught.code, StatusCode.UNIMPLEMENTED) + self.assertEqual( + exc_caught.details, + 'BigtableTableService.RenameTable is not yet implemented') + def test_create_column_family(self): temp_table_id = 'foo-bar-baz-table' temp_table = CLUSTER.table(temp_table_id) @@ -224,6 +244,29 @@ def test_create_column_family(self): column_family.column_family_id) self.assertEqual(retrieved_col_fam.gc_rule, gc_rule) + def test_update_column_family(self): + temp_table_id = 'foo-bar-baz-table' + temp_table = CLUSTER.table(temp_table_id) + temp_table.create() + self.tables_to_delete.append(temp_table) + + gc_rule = GarbageCollectionRule(max_num_versions=1) + column_family = temp_table.column_family(COLUMN_FAMILY_ID1, + gc_rule=gc_rule) + column_family.create() + + # Check that our created table is as expected. + col_fams = temp_table.list_column_families() + self.assertEqual(col_fams, {COLUMN_FAMILY_ID1: column_family}) + + # Update the column family's GC rule and then try to update. + column_family.gc_rule = None + column_family.update() + + # Check that the update has propagated. + col_fams = temp_table.list_column_families() + self.assertEqual(col_fams[COLUMN_FAMILY_ID1].gc_rule, None) + def test_delete_column_family(self): temp_table_id = 'foo-bar-baz-table' temp_table = CLUSTER.table(temp_table_id) From b2741d65ccfdff9c39339549ef0db961f0ceaf23 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 16:33:22 -0800 Subject: [PATCH 30/65] Bringing local client.py closer to upstream. This is so that a simple diff is possible. --- gcloud_bigtable/_non_upstream_helpers.py | 215 ++++++++++++ gcloud_bigtable/client.py | 243 +------------- gcloud_bigtable/test__non_upstream_helpers.py | 307 ++++++++++++++++++ gcloud_bigtable/test_client.py | 304 +---------------- 4 files changed, 552 insertions(+), 517 deletions(-) create mode 100644 gcloud_bigtable/_non_upstream_helpers.py create mode 100644 gcloud_bigtable/test__non_upstream_helpers.py diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py new file mode 100644 index 0000000..5fb4151 --- /dev/null +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -0,0 +1,215 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers needed that are already present in gcloud-python. + +They are only needed here, but not needed for merging this upstream. +""" + + +import os +import socket + +import six + +from oauth2client.client import GoogleCredentials +from oauth2client.client import SignedJwtAssertionCredentials +from oauth2client.client import _get_application_default_credential_from_file + +try: + from google.appengine.api import app_identity +except ImportError: + app_identity = None + + +PROJECT_ENV_VAR = 'GCLOUD_PROJECT' +"""Environment variable used to provide an implicit project ID.""" + + +def _project_from_environment(): + """Attempts to get the project ID from an environment variable. + + :rtype: :class:`str` or :data:`NoneType ` + :returns: The project ID provided or :data:`None` + """ + return os.getenv(PROJECT_ENV_VAR) + + +def _project_from_app_engine(): + """Gets the App Engine application ID if it can be inferred. + + :rtype: :class:`str` or :data:`NoneType ` + :returns: App Engine application ID if running in App Engine, + else :data:`None`. + """ + if app_identity is None: + return None + + return app_identity.get_application_id() + + +def _project_from_compute_engine(): + """Gets the Compute Engine project ID if it can be inferred. + + Uses 169.254.169.254 for the metadata server to avoid request + latency from DNS lookup. + + See https://cloud.google.com/compute/docs/metadata#metadataserver + for information about this IP address. (This IP is also used for + Amazon EC2 instances, so the metadata flavor is crucial.) + + See https://github.com/google/oauth2client/issues/93 for context about + DNS latency. + + :rtype: :class:`str` or :data:`NoneType ` + :returns: Compute Engine project ID if the metadata service is available, + else :data:`None`. + """ + host = '169.254.169.254' + uri_path = '/computeMetadata/v1/project/project-id' + headers = {'Metadata-Flavor': 'Google'} + connection = six.moves.http_client.HTTPConnection(host, timeout=0.1) + + try: + connection.request('GET', uri_path, headers=headers) + response = connection.getresponse() + if response.status == 200: + return response.read() + except socket.error: # socket.timeout or socket.error(64, 'Host is down') + pass + finally: + connection.close() + + +def _determine_project(project=None): + """Determine the project ID from the input or environment. + + When checking the environment, the following precedence is observed: + + * GCLOUD_PROJECT environment variable + * Google App Engine application ID + * Google Compute Engine project ID (from metadata server) + + :type project: str + :param project: (Optional) The ID of the project which owns the + clusters, tables and data. If not provided, will attempt + to determine from the environment. + + :rtype: str + :returns: The project ID provided or inferred from the environment. + :raises: :class:`EnvironmentError` if the project ID was not + passed in and can't be inferred from the environment. + """ + if project is None: + project = _project_from_environment() + + if project is None: + project = _project_from_app_engine() + + if project is None: + project = _project_from_compute_engine() + + if project is None: + raise EnvironmentError('Project ID was not provided and could not ' + 'be determined from environment.') + + return project + + +def _get_contents(filename): + """Get the contents of a file. + + This is just implemented so we can stub out while testing. + + :type filename: :class:`str` or :func:`unicode ` + :param filename: The name of a file to open. + + :rtype: bytes + :returns: The bytes loaded from the file. + """ + with open(filename, 'rb') as file_obj: + return file_obj.read() + + +def get_credentials(): + """Get default credentials.""" + return GoogleCredentials.get_application_default() + + +class _FactoryMixin(object): + """Mixin for factories to get credentials.""" + + def __init__(self, project=None, credentials=None): + self.project = _determine_project(project) + self._credentials = credentials + + @classmethod + def from_service_account_json(cls, json_credentials_path, *args, **kwargs): + """Factory to retrieve JSON credentials while creating client object. + + :type json_credentials_path: str + :param json_credentials_path: The path to a private key file (this file + was given to you when you created the + service account). This file must contain + a JSON object with a private key and + other credentials information (downloaded + from the Google APIs console). + + :type args: tuple + :param args: Positional arguments. + + :type kwargs: dict + :param kwargs: Keyword arguments. + + :rtype: :class:`_FactoryMixin` + :returns: The client created with the retrieved JSON credentials. + """ + credentials = _get_application_default_credential_from_file( + json_credentials_path) + kwargs['credentials'] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_p12(cls, client_email, private_key_path, + *args, **kwargs): + """Factory to retrieve P12 credentials while creating client object. + + .. note:: + + Unless you have an explicit reason to use a PKCS12 key for your + service account, we recommend using a JSON key. + + :type client_email: str + :param client_email: The e-mail attached to the service account. + + :type private_key_path: str + :param private_key_path: The path to a private key file (this file was + given to you when you created the service + account). This file must be in P12 format. + + :type args: tuple + :param args: Positional arguments. + + :type kwargs: dict + :param kwargs: Keyword arguments. + + :rtype: :class:`_FactoryMixin` + :returns: The client created with the retrieved P12 credentials. + """ + credentials = SignedJwtAssertionCredentials( + service_account_name=client_email, + private_key=_get_contents(private_key_path), + scope=None) + kwargs['credentials'] = credentials + return cls(*args, **kwargs) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 4b98107..3cc39a5 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -28,19 +28,6 @@ import copy -import os -import socket - -import six - -from oauth2client.client import GoogleCredentials -from oauth2client.client import SignedJwtAssertionCredentials -from oauth2client.client import _get_application_default_credential_from_file - -try: - from google.appengine.api import app_identity -except ImportError: - app_identity = None from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( @@ -50,6 +37,8 @@ from gcloud_bigtable._generated import bigtable_table_service_pb2 from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._helpers import make_stub +from gcloud_bigtable._non_upstream_helpers import _FactoryMixin +from gcloud_bigtable._non_upstream_helpers import get_credentials from gcloud_bigtable.cluster import Cluster @@ -83,9 +72,6 @@ 'cloud-bigtable.data.readonly') """Scope for reading table data.""" -PROJECT_ENV_VAR = 'GCLOUD_PROJECT' -"""Environment variable used to provide an implicit project ID.""" - DEFAULT_TIMEOUT_SECONDS = 10 """The default timeout to use for API requests.""" @@ -93,97 +79,7 @@ """The default user agent for API requests.""" -def _project_from_environment(): - """Attempts to get the project ID from an environment variable. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: The project ID provided or :data:`None` - """ - return os.getenv(PROJECT_ENV_VAR) - - -def _project_from_app_engine(): - """Gets the App Engine application ID if it can be inferred. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: App Engine application ID if running in App Engine, - else :data:`None`. - """ - if app_identity is None: - return None - - return app_identity.get_application_id() - - -def _project_from_compute_engine(): - """Gets the Compute Engine project ID if it can be inferred. - - Uses 169.254.169.254 for the metadata server to avoid request - latency from DNS lookup. - - See https://cloud.google.com/compute/docs/metadata#metadataserver - for information about this IP address. (This IP is also used for - Amazon EC2 instances, so the metadata flavor is crucial.) - - See https://github.com/google/oauth2client/issues/93 for context about - DNS latency. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: Compute Engine project ID if the metadata service is available, - else :data:`None`. - """ - host = '169.254.169.254' - uri_path = '/computeMetadata/v1/project/project-id' - headers = {'Metadata-Flavor': 'Google'} - connection = six.moves.http_client.HTTPConnection(host, timeout=0.1) - - try: - connection.request('GET', uri_path, headers=headers) - response = connection.getresponse() - if response.status == 200: - return response.read() - except socket.error: # socket.timeout or socket.error(64, 'Host is down') - pass - finally: - connection.close() - - -def _determine_project(project=None): - """Determine the project ID from the input or environment. - - When checking the environment, the following precedence is observed: - - * GCLOUD_PROJECT environment variable - * Google App Engine application ID - * Google Compute Engine project ID (from metadata server) - - :type project: str - :param project: (Optional) The ID of the project which owns the - clusters, tables and data. If not provided, will attempt - to determine from the environment. - - :rtype: str - :returns: The project ID provided or inferred from the environment. - :raises: :class:`EnvironmentError` if the project ID was not - passed in and can't be inferred from the environment. - """ - if project is None: - project = _project_from_environment() - - if project is None: - project = _project_from_app_engine() - - if project is None: - project = _project_from_compute_engine() - - if project is None: - raise EnvironmentError('Project ID was not provided and could not ' - 'be determined from environment.') - - return project - - -class Client(object): +class Client(_FactoryMixin): """Client for interacting with Google Cloud Bigtable API. .. note:: @@ -229,9 +125,9 @@ class Client(object): def __init__(self, project=None, credentials=None, read_only=False, admin=False, user_agent=DEFAULT_USER_AGENT, timeout_seconds=DEFAULT_TIMEOUT_SECONDS): - self.project = _determine_project(project) + super(Client, self).__init__(project=project) if credentials is None: - credentials = GoogleCredentials.get_application_default() + credentials = get_credentials() if read_only and admin: raise ValueError('A read-only client cannot also perform' @@ -257,85 +153,6 @@ def __init__(self, project=None, credentials=None, self._operations_stub_internal = None self._table_stub_internal = None - @classmethod - def from_service_account_json(cls, json_credentials_path, project=None, - read_only=False, admin=False): - """Factory to retrieve JSON credentials while creating client object. - - :type json_credentials_path: str - :param json_credentials_path: The path to a private key file (this file - was given to you when you created the - service account). This file must contain - a JSON object with a private key and - other credentials information (downloaded - from the Google APIs console). - - :type project: str - :param project: The ID of the project which owns the clusters, - tables and data. Will be passed to :class:`Client` - constructor. - - :type read_only: bool - :param read_only: Boolean indicating if the data scope should be - for reading only (or for writing as well). Will be - passed to :class:`Client` constructor. - - :type admin: bool - :param admin: Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. Will - be passed to :class:`Client` constructor. - - :rtype: :class:`Client` - :returns: The client created with the retrieved JSON credentials. - """ - credentials = _get_application_default_credential_from_file( - json_credentials_path) - return cls(credentials=credentials, project=project, - read_only=read_only, admin=admin) - - @classmethod - def from_service_account_p12(cls, client_email, private_key_path, - project=None, read_only=False, - admin=False): - """Factory to retrieve P12 credentials while creating client object. - - .. note:: - Unless you have an explicit reason to use a PKCS12 key for your - service account, we recommend using a JSON key. - - :type client_email: str - :param client_email: The e-mail attached to the service account. - - :type private_key_path: str - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). This file must be in P12 format. - - :type project: str - :param project: The ID of the project which owns the clusters, - tables and data. Will be passed to :class:`Client` - constructor. - - :type read_only: bool - :param read_only: Boolean indicating if the data scope should be - for reading only (or for writing as well). Will be - passed to :class:`Client` constructor. - - :type admin: bool - :param admin: Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. Will - be passed to :class:`Client` constructor. - - :rtype: :class:`Client` - :returns: The client created with the retrieved P12 credentials. - """ - credentials = SignedJwtAssertionCredentials( - service_account_name=client_email, - private_key=_get_contents(private_key_path), - scope=None) - return cls(credentials=credentials, project=project, - read_only=read_only, admin=admin) - def copy(self): """Make a copy of this client. @@ -370,8 +187,9 @@ def project_name(self): """Project name to be used with Cluster Admin API. .. note:: - This property will not change if ``project`` does not, but the - return value is not cached. + + This property will not change if ``project`` does not, but the + return value is not cached. The project name is of the form @@ -387,7 +205,7 @@ def project_name(self): def _data_stub(self): """Getter for the gRPC stub used for the Data API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client has not been :meth:`start`-ed. @@ -400,7 +218,7 @@ def _data_stub(self): def _cluster_stub(self): """Getter for the gRPC stub used for the Cluster Admin API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -416,7 +234,7 @@ def _cluster_stub(self): def _operations_stub(self): """Getter for the gRPC stub used for the Operations API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -432,7 +250,7 @@ def _operations_stub(self): def _table_stub(self): """Getter for the gRPC stub used for the Table Admin API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. :raises: :class:`ValueError ` if the current client is not an admin client or if it has not been @@ -447,7 +265,7 @@ def _table_stub(self): def _make_data_stub(self): """Creates gRPC stub to make requests to the Data API. - :rtype: :class:`grpc.early_adopter.implementations._Stub` + :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ return make_stub(self, DATA_STUB_FACTORY, @@ -559,23 +377,18 @@ def cluster(self, zone, cluster_id, display_name=None, serve_nodes=3): return Cluster(zone, cluster_id, self, display_name=display_name, serve_nodes=serve_nodes) - def list_zones(self, timeout_seconds=None): + def list_zones(self): """Lists zones associated with project. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on client. - :rtype: list :returns: The names (as :class:`str`) of the zones :raises: :class:`ValueError ` if one of the zones is not in ``OK`` state. """ request_pb = messages_pb2.ListZonesRequest(name=self.project_name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.messages_pb2.ListZonesResponse` - list_zones_response = self._cluster_stub.ListZones(request_pb, - timeout_seconds) + list_zones_response = self._cluster_stub.ListZones( + request_pb, self.timeout_seconds) result = [] for zone in list_zones_response.zones: @@ -585,41 +398,21 @@ def list_zones(self, timeout_seconds=None): result.append(zone.display_name) return result - def list_clusters(self, timeout_seconds=None): + def list_clusters(self): """Lists clusters owned by the project. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on client. - :rtype: tuple :returns: A pair of results, the first is a list of :class:`.Cluster` s returned and the second is a list of strings (the failed zones in the request). """ request_pb = messages_pb2.ListClustersRequest(name=self.project_name) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.messages_pb2.ListClustersResponse` list_clusters_response = self._cluster_stub.ListClusters( - request_pb, timeout_seconds) + request_pb, self.timeout_seconds) failed_zones = [zone.display_name for zone in list_clusters_response.failed_zones] clusters = [Cluster.from_pb(cluster_pb, self) for cluster_pb in list_clusters_response.clusters] return clusters, failed_zones - - -def _get_contents(filename): - """Get the contents of a file. - - This is just implemented so we can stub out while testing. - - :type filename: :class:`str` or :func:`unicode ` - :param filename: The name of a file to open. - - :rtype: bytes - :returns: The bytes loaded from the file. - """ - with open(filename, 'rb') as file_obj: - return file_obj.read() diff --git a/gcloud_bigtable/test__non_upstream_helpers.py b/gcloud_bigtable/test__non_upstream_helpers.py new file mode 100644 index 0000000..426e3d0 --- /dev/null +++ b/gcloud_bigtable/test__non_upstream_helpers.py @@ -0,0 +1,307 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest2 + + +PROJECT = 'project-id' + + +class Test__project_from_environment(unittest2.TestCase): + + def _callFUT(self): + from gcloud_bigtable._non_upstream_helpers import ( + _project_from_environment) + return _project_from_environment() + + def test_it(self): + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + fake_project = object() + mock_os = _MockWithAttachedMethods(fake_project) + with _Monkey(MUT, os=mock_os): + result = self._callFUT() + + self.assertTrue(result is fake_project) + self.assertEqual(mock_os._called, + [('getenv', (MUT.PROJECT_ENV_VAR,), {})]) + + +class Test__project_from_app_engine(unittest2.TestCase): + + def _callFUT(self): + from gcloud_bigtable._non_upstream_helpers import ( + _project_from_app_engine) + return _project_from_app_engine() + + def test_without_app_engine(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + with _Monkey(MUT, app_identity=None): + result = self._callFUT() + + self.assertEqual(result, None) + + def test_with_app_engine(self): + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + fake_project = object() + mock_app_identity = _MockWithAttachedMethods(fake_project) + with _Monkey(MUT, app_identity=mock_app_identity): + result = self._callFUT() + + self.assertTrue(result is fake_project) + self.assertEqual(mock_app_identity._called, + [('get_application_id', (), {})]) + + +class Test__project_from_compute_engine(unittest2.TestCase): + + def _callFUT(self): + from gcloud_bigtable._non_upstream_helpers import ( + _project_from_compute_engine) + return _project_from_compute_engine() + + @staticmethod + def _make_http_connection_response(status_code, read_result, + raise_socket_err=False): + import socket + + class Response(object): + status = status_code + + @staticmethod + def read(): + if raise_socket_err: + raise socket.error('Failed') + else: + return read_result + return Response + + @staticmethod + def _make_fake_six_module(mock_http_client): + class MockSix(object): + class moves(object): + http_client = mock_http_client + return MockSix + + def _helper(self, status, raise_socket_err=False): + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + fake_project = object() + response = self._make_http_connection_response( + status, fake_project, raise_socket_err=raise_socket_err) + # The connection does the bulk of the work. + mock_connection = _MockWithAttachedMethods(None, response, None) + # The http_client module holds the connection constructor. + mock_http_client = _MockWithAttachedMethods(mock_connection) + # We need to put the client in place of it's location in six. + mock_six = self._make_fake_six_module(mock_http_client) + + with _Monkey(MUT, six=mock_six): + result = self._callFUT() + + if status == 200 and not raise_socket_err: + self.assertEqual(result, fake_project) + else: + self.assertEqual(result, None) + + self.assertEqual(mock_connection._called, [ + ( + 'request', + ('GET', '/computeMetadata/v1/project/project-id'), + {'headers': {'Metadata-Flavor': 'Google'}}, + ), + ( + 'getresponse', + (), + {}, + ), + ( + 'close', + (), + {}, + ), + ]) + self.assertEqual(mock_http_client._called, [ + ( + 'HTTPConnection', + ('169.254.169.254',), + {'timeout': 0.1}, + ), + ]) + + def test_success(self): + self._helper(200) + + def test_failed_status(self): + self._helper(404) + + def test_read_fails_with_socket_error(self): + self._helper(200, raise_socket_err=True) + + +class Test__determine_project(unittest2.TestCase): + + def _callFUT(self, project): + from gcloud_bigtable._non_upstream_helpers import _determine_project + return _determine_project(project) + + def _helper(self, num_mocks_called, mock_output, method_input): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + mock_project_from_environment = _MockCalled(None) + mock_project_from_app_engine = _MockCalled(None) + mock_project_from_compute_engine = _MockCalled(None) + + monkey_kwargs = { + '_project_from_environment': mock_project_from_environment, + '_project_from_app_engine': mock_project_from_app_engine, + '_project_from_compute_engine': ( + mock_project_from_compute_engine), + } + # Need the mocks in order they are called, so we can + # access them based on `num_mocks_called`. + mocks = [ + mock_project_from_environment, + mock_project_from_app_engine, + mock_project_from_compute_engine, + ] + mocks[num_mocks_called - 1].result = mock_output + + with _Monkey(MUT, **monkey_kwargs): + if num_mocks_called == 3 and mock_output is None: + with self.assertRaises(EnvironmentError): + self._callFUT(method_input) + else: + result = self._callFUT(method_input) + self.assertEqual(result, method_input or mock_output) + + # Make sure our mocks were called with no arguments. + for mock in mocks[:num_mocks_called]: + mock.check_called(self, [()]) + for mock in mocks[num_mocks_called:]: + mock.check_called(self, []) + + def test_fail_to_infer(self): + self._helper(num_mocks_called=3, mock_output=None, + method_input=None) + + def test_with_explicit_value(self): + self._helper(num_mocks_called=0, mock_output=None, + method_input=PROJECT) + + def test_from_environment(self): + self._helper(num_mocks_called=1, mock_output=PROJECT, + method_input=None) + + def test_from_app_engine(self): + self._helper(num_mocks_called=2, mock_output=PROJECT, + method_input=None) + + def test_from_compute_engine(self): + self._helper(num_mocks_called=3, mock_output=PROJECT, + method_input=None) + + +class Test__get_contents(unittest2.TestCase): + + def _callFUT(self, filename): + from gcloud_bigtable._non_upstream_helpers import _get_contents + return _get_contents(filename) + + def test_it(self): + import tempfile + + filename = tempfile.mktemp() + contents = b'foobar' + with open(filename, 'wb') as file_obj: + file_obj.write(contents) + + self.assertEqual(self._callFUT(filename), contents) + + +class Test__FactoryMixin(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable._non_upstream_helpers import _FactoryMixin + return _FactoryMixin + + def test_from_service_account_json(self): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + klass = self._getTargetClass() + scoped_creds = object() + credentials = _MockWithAttachedMethods(scoped_creds) + get_adc = _MockCalled(credentials) + json_credentials_path = 'JSON_CREDENTIALS_PATH' + + with _Monkey(MUT, + _get_application_default_credential_from_file=get_adc): + client = klass.from_service_account_json( + json_credentials_path, project=PROJECT) + + self.assertEqual(client.project, PROJECT) + self.assertTrue(client._credentials is credentials) + + self.assertEqual(credentials._called, []) + # _get_application_default_credential_from_file only has pos. args. + get_adc.check_called(self, [(json_credentials_path,)]) + + def test_from_service_account_p12(self): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import _non_upstream_helpers as MUT + + klass = self._getTargetClass() + scoped_creds = object() + credentials = _MockWithAttachedMethods(scoped_creds) + signed_creds = _MockCalled(credentials) + + private_key = 'PRIVATE_KEY' + mock_get_contents = _MockCalled(private_key) + client_email = 'CLIENT_EMAIL' + private_key_path = 'PRIVATE_KEY_PATH' + + with _Monkey(MUT, SignedJwtAssertionCredentials=signed_creds, + _get_contents=mock_get_contents): + client = klass.from_service_account_p12( + client_email, private_key_path, project=PROJECT) + + self.assertEqual(client.project, PROJECT) + self.assertTrue(client._credentials is credentials) + self.assertEqual(credentials._called, []) + # SignedJwtAssertionCredentials() called with only kwargs + signed_creds_kw = { + 'private_key': private_key, + 'service_account_name': client_email, + 'scope': None, + } + signed_creds.check_called(self, [()], [signed_creds_kw]) + # Load private key (via _get_contents) from the key path. + mock_get_contents.check_called(self, [(private_key_path,)]) diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index d3803a9..cbf0579 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -19,209 +19,6 @@ PROJECT = 'project-id' -class Test__project_from_environment(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_environment - return _project_from_environment() - - def test_it(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - mock_os = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, os=mock_os): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_os._called, - [('getenv', (MUT.PROJECT_ENV_VAR,), {})]) - - -class Test__project_from_app_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_app_engine - return _project_from_app_engine() - - def test_without_app_engine(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - with _Monkey(MUT, app_identity=None): - result = self._callFUT() - - self.assertEqual(result, None) - - def test_with_app_engine(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - mock_app_identity = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, app_identity=mock_app_identity): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_app_identity._called, - [('get_application_id', (), {})]) - - -class Test__project_from_compute_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_compute_engine - return _project_from_compute_engine() - - @staticmethod - def _make_http_connection_response(status_code, read_result, - raise_socket_err=False): - import socket - - class Response(object): - status = status_code - - @staticmethod - def read(): - if raise_socket_err: - raise socket.error('Failed') - else: - return read_result - return Response - - @staticmethod - def _make_fake_six_module(mock_http_client): - class MockSix(object): - class moves(object): - http_client = mock_http_client - return MockSix - - def _helper(self, status, raise_socket_err=False): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - response = self._make_http_connection_response( - status, fake_project, raise_socket_err=raise_socket_err) - # The connection does the bulk of the work. - mock_connection = _MockWithAttachedMethods(None, response, None) - # The http_client module holds the connection constructor. - mock_http_client = _MockWithAttachedMethods(mock_connection) - # We need to put the client in place of it's location in six. - mock_six = self._make_fake_six_module(mock_http_client) - - with _Monkey(MUT, six=mock_six): - result = self._callFUT() - - if status == 200 and not raise_socket_err: - self.assertEqual(result, fake_project) - else: - self.assertEqual(result, None) - - self.assertEqual(mock_connection._called, [ - ( - 'request', - ('GET', '/computeMetadata/v1/project/project-id'), - {'headers': {'Metadata-Flavor': 'Google'}}, - ), - ( - 'getresponse', - (), - {}, - ), - ( - 'close', - (), - {}, - ), - ]) - self.assertEqual(mock_http_client._called, [ - ( - 'HTTPConnection', - ('169.254.169.254',), - {'timeout': 0.1}, - ), - ]) - - def test_success(self): - self._helper(200) - - def test_failed_status(self): - self._helper(404) - - def test_read_fails_with_socket_error(self): - self._helper(200, raise_socket_err=True) - - -class Test__determine_project(unittest2.TestCase): - - def _callFUT(self, project): - from gcloud_bigtable.client import _determine_project - return _determine_project(project) - - def _helper(self, num_mocks_called, mock_output, method_input): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - mock_project_from_environment = _MockCalled(None) - mock_project_from_app_engine = _MockCalled(None) - mock_project_from_compute_engine = _MockCalled(None) - - monkey_kwargs = { - '_project_from_environment': mock_project_from_environment, - '_project_from_app_engine': mock_project_from_app_engine, - '_project_from_compute_engine': ( - mock_project_from_compute_engine), - } - # Need the mocks in order they are called, so we can - # access them based on `num_mocks_called`. - mocks = [ - mock_project_from_environment, - mock_project_from_app_engine, - mock_project_from_compute_engine, - ] - mocks[num_mocks_called - 1].result = mock_output - - with _Monkey(MUT, **monkey_kwargs): - if num_mocks_called == 3 and mock_output is None: - with self.assertRaises(EnvironmentError): - self._callFUT(method_input) - else: - result = self._callFUT(method_input) - self.assertEqual(result, method_input or mock_output) - - # Make sure our mocks were called with no arguments. - for mock in mocks[:num_mocks_called]: - mock.check_called(self, [()]) - for mock in mocks[num_mocks_called:]: - mock.check_called(self, []) - - def test_fail_to_infer(self): - self._helper(num_mocks_called=3, mock_output=None, - method_input=None) - - def test_with_explicit_value(self): - self._helper(num_mocks_called=0, mock_output=None, - method_input=PROJECT) - - def test_from_environment(self): - self._helper(num_mocks_called=1, mock_output=PROJECT, - method_input=None) - - def test_from_app_engine(self): - self._helper(num_mocks_called=2, mock_output=PROJECT, - method_input=None) - - def test_from_compute_engine(self): - self._helper(num_mocks_called=3, mock_output=PROJECT, - method_input=None) - - class TestClient(unittest2.TestCase): def _getTargetClass(self): @@ -234,6 +31,7 @@ def _makeOne(self, *args, **kwargs): def _constructor_test_helper(self, expected_scopes, project=None, read_only=False, admin=False, user_agent=None): + from gcloud_bigtable import _non_upstream_helpers as OTHER_MOD from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _MockWithAttachedMethods from gcloud_bigtable._testing import _Monkey @@ -243,7 +41,7 @@ def _constructor_test_helper(self, expected_scopes, project=None, credentials = _MockWithAttachedMethods(scoped_creds) determined_project = object() mock_determine_project = _MockCalled(determined_project) - with _Monkey(MUT, _determine_project=mock_determine_project): + with _Monkey(OTHER_MOD, _determine_project=mock_determine_project): client = self._makeOne(project=project, credentials=credentials, read_only=read_only, admin=admin, user_agent=user_agent) @@ -258,6 +56,7 @@ def _constructor_test_helper(self, expected_scopes, project=None, mock_determine_project.check_called(self, [(project,)]) def test_constructor_default(self): + from gcloud_bigtable import _non_upstream_helpers as OTHER_MOD from gcloud_bigtable._testing import _MockWithAttachedMethods from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import client as MUT @@ -266,7 +65,7 @@ def test_constructor_default(self): credentials = _MockWithAttachedMethods(scoped_creds) mock_creds_class = _MockWithAttachedMethods(credentials) - with _Monkey(MUT, GoogleCredentials=mock_creds_class): + with _Monkey(OTHER_MOD, GoogleCredentials=mock_creds_class): client = self._makeOne(project=PROJECT) self.assertEqual(client.project, PROJECT) @@ -310,70 +109,6 @@ def test_constructor_both_admin_and_read_only(self): self._makeOne(project=PROJECT, credentials=None, admin=True, read_only=True) - def test_from_service_account_json(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - get_adc = _MockCalled(credentials) - json_credentials_path = 'JSON_CREDENTIALS_PATH' - - with _Monkey(MUT, - _get_application_default_credential_from_file=get_adc): - client = klass.from_service_account_json( - json_credentials_path, project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - # _get_application_default_credential_from_file only has pos. args. - get_adc.check_called(self, [(json_credentials_path,)]) - - def test_from_service_account_p12(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - signed_creds = _MockCalled(credentials) - - private_key = 'PRIVATE_KEY' - mock_get_contents = _MockCalled(private_key) - client_email = 'CLIENT_EMAIL' - private_key_path = 'PRIVATE_KEY_PATH' - - with _Monkey(MUT, SignedJwtAssertionCredentials=signed_creds, - _get_contents=mock_get_contents): - client = klass.from_service_account_p12( - client_email, private_key_path, project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - # SignedJwtAssertionCredentials() called with only kwargs - signed_creds_kw = { - 'private_key': private_key, - 'service_account_name': client_email, - 'scope': None, - } - signed_creds.check_called(self, [()], [signed_creds_kw]) - # Load private key (via _get_contents) from the key path. - mock_get_contents.check_called(self, [(private_key_path,)]) - def _copy_test_helper(self, read_only=False, admin=False): class Credentials(object): @@ -788,8 +523,10 @@ def _list_zones_helper(self, zone_status): scoped_creds = object() credentials = _MockWithAttachedMethods(scoped_creds) + timeout_seconds = 281330 client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) + credentials=credentials, admin=True, + timeout_seconds=timeout_seconds) # Create request_pb request_pb = messages_pb2.ListZonesRequest( @@ -813,8 +550,7 @@ def _list_zones_helper(self, zone_status): expected_result = [zone1, zone2] # Perform the method and check the result. - timeout_seconds = 281330 - result = client.list_zones(timeout_seconds=timeout_seconds) + result = client.list_zones() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListZones', @@ -843,8 +579,10 @@ def test_list_clusters(self): scoped_creds = object() credentials = _MockWithAttachedMethods(scoped_creds) + timeout_seconds = 8004 client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) + credentials=credentials, admin=True, + timeout_seconds=timeout_seconds) # Create request_pb request_pb = messages_pb2.ListClustersRequest( @@ -890,8 +628,7 @@ def test_list_clusters(self): expected_result = (clusters, failed_zones) # Perform the method and check the result. - timeout_seconds = 8004 - result = client.list_clusters(timeout_seconds=timeout_seconds) + result = client.list_clusters() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'ListClusters', @@ -900,23 +637,6 @@ def test_list_clusters(self): )]) -class Test__get_contents(unittest2.TestCase): - - def _callFUT(self, filename): - from gcloud_bigtable.client import _get_contents - return _get_contents(filename) - - def test_it(self): - import tempfile - - filename = tempfile.mktemp() - contents = b'foobar' - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - self.assertEqual(self._callFUT(filename), contents) - - class _FakeStub(object): def __init__(self): From 8c11909a811e7329a36d4a73d347106340e1eead Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 16:46:07 -0800 Subject: [PATCH 31/65] Bringing local cluster.py closer to upstream. This is so that a simple diff is possible. --- gcloud_bigtable/_helpers.py | 56 +----------------------- gcloud_bigtable/cluster.py | 54 ++++++++++++++++++++++- gcloud_bigtable/test__helpers.py | 74 -------------------------------- gcloud_bigtable/test_cluster.py | 74 ++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 131 deletions(-) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index f9225e0..e971b1e 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Utility methods for gcloud_bigtable. +"""Utility methods for gcloud Bigtable. Primarily includes helpers for dealing with low-level protobuf objects. @@ -27,23 +27,9 @@ from grpc.beta import implementations -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import duration_pb2 -_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' -_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' -_CLUSTER_TYPE_URL = _ADMIN_TYPE_URL_BASE + 'Cluster' -_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' -_TYPE_URL_MAP = { - _CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata, - _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata': ( - messages_pb2.UndeleteClusterMetadata), - _ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata': ( - messages_pb2.UpdateClusterMetadata), -} - EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) # See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 # for appropriate values on other systems. @@ -82,46 +68,6 @@ class AuthInfo(object): ROOT_CERTIFICATES = None -def _pb_timestamp_to_datetime(timestamp): - """Convert a Timestamp protobuf to a datetime object. - - :type timestamp: :class:`._generated.timestamp_pb2.Timestamp` - :param timestamp: A Google returned timestamp protobuf. - - :rtype: :class:`datetime.datetime` - :returns: A UTC datetime object converted from a protobuf timestamp. - """ - return ( - EPOCH + - datetime.timedelta( - seconds=timestamp.seconds, - microseconds=(timestamp.nanos / 1000.0), - ) - ) - - -def _parse_pb_any_to_native(any_val, expected_type=None): - """Convert a serialized "google.protobuf.Any" value to actual type. - - :type any_val: :class:`gcloud_bigtable._generated.any_pb2.Any` - :param any_val: A serialized protobuf value container. - - :type expected_type: str - :param expected_type: (Optional) The type URL we expect ``any_val`` - to have. - - :rtype: object - :returns: The de-serialized object. - :raises: :class:`ValueError ` if the - ``expected_type`` does not match the ``type_url`` on the input. - """ - if expected_type is not None and expected_type != any_val.type_url: - raise ValueError('Expected type: %s, Received: %s' % ( - expected_type, any_val.type_url)) - container_class = _TYPE_URL_MAP[any_val.type_url] - return container_class.FromString(any_val.value) - - def _timedelta_to_duration_pb(timedelta_val): """Convert a Python timedelta object to a duration protobuf. diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 2035d44..98b2b84 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -15,6 +15,7 @@ """User friendly container for Google Cloud Bigtable Cluster.""" +import datetime import re from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 @@ -23,8 +24,7 @@ from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import _parse_pb_any_to_native -from gcloud_bigtable._helpers import _pb_timestamp_to_datetime +from gcloud_bigtable._helpers import EPOCH as _EPOCH from gcloud_bigtable.table import Table @@ -35,6 +35,16 @@ r'clusters/([a-z][-a-z0-9]*)/operations/' r'(?P\d+)$') _DEFAULT_SERVE_NODES = 3 +_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' +_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' +_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' +_UPDATE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata' +_UNDELETE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata' +_TYPE_URL_MAP = { + _CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata, + _UPDATE_CREATE_METADATA: messages_pb2.UpdateClusterMetadata, + _UNDELETE_CREATE_METADATA: messages_pb2.UndeleteClusterMetadata, +} def _get_pb_property_value(message_pb, property_name): @@ -82,6 +92,46 @@ def _prepare_create_request(cluster): ) +def _pb_timestamp_to_datetime(timestamp): + """Convert a Timestamp protobuf to a datetime object. + + :type timestamp: :class:`._generated.timestamp_pb2.Timestamp` + :param timestamp: A Google returned timestamp protobuf. + + :rtype: :class:`datetime.datetime` + :returns: A UTC datetime object converted from a protobuf timestamp. + """ + return ( + _EPOCH + + datetime.timedelta( + seconds=timestamp.seconds, + microseconds=(timestamp.nanos / 1000.0), + ) + ) + + +def _parse_pb_any_to_native(any_val, expected_type=None): + """Convert a serialized "google.protobuf.Any" value to actual type. + + :type any_val: :class:`gcloud.bigtable._generated.any_pb2.Any` + :param any_val: A serialized protobuf value container. + + :type expected_type: str + :param expected_type: (Optional) The type URL we expect ``any_val`` + to have. + + :rtype: object + :returns: The de-serialized object. + :raises: :class:`ValueError ` if the + ``expected_type`` does not match the ``type_url`` on the input. + """ + if expected_type is not None and expected_type != any_val.type_url: + raise ValueError('Expected type: %s, Received: %s' % ( + expected_type, any_val.type_url)) + container_class = _TYPE_URL_MAP[any_val.type_url] + return container_class.FromString(any_val.value) + + def _process_operation(operation_pb): """Processes a create protobuf response. diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 06107b3..7fe0377 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -73,80 +73,6 @@ class _ReturnVal(object): self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) -class Test__pb_timestamp_to_datetime(unittest2.TestCase): - - def _callFUT(self, timestamp): - from gcloud_bigtable._helpers import _pb_timestamp_to_datetime - return _pb_timestamp_to_datetime(timestamp) - - def test_it(self): - import datetime - import pytz - from gcloud_bigtable._generated.timestamp_pb2 import Timestamp - - # Epoch is midnight on January 1, 1970 ... - dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0, - minute=1, second=1, microsecond=1234, - tzinfo=pytz.utc) - # ... so 1 minute and 1 second after is 61 seconds and 1234 - # microseconds is 1234000 nanoseconds. - timestamp = Timestamp(seconds=61, nanos=1234000) - self.assertEqual(self._callFUT(timestamp), dt_stamp) - - -class Test__parse_pb_any_to_native(unittest2.TestCase): - - def _callFUT(self, any_val, expected_type=None): - from gcloud_bigtable._helpers import _parse_pb_any_to_native - return _parse_pb_any_to_native(any_val, expected_type=expected_type) - - def test_it(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name - fake_type_url_map = {type_url: data_pb2.Cell} - - cell = data_pb2.Cell( - timestamp_micros=0, - value=b'foobar', - ) - any_val = any_pb2.Any( - type_url=type_url, - value=cell.SerializeToString(), - ) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - result = self._callFUT(any_val) - - self.assertEqual(result, cell) - - def test_unknown_type_url(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - fake_type_url_map = {} - any_val = any_pb2.Any() - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(KeyError): - self._callFUT(any_val) - - def test_disagreeing_type_url(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - type_url1 = 'foo' - type_url2 = 'bar' - fake_type_url_map = {type_url1: None} - any_val = any_pb2.Any(type_url=type_url2) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(ValueError): - self._callFUT(any_val, expected_type=type_url1) - - class Test__timedelta_to_duration_pb(unittest2.TestCase): def _callFUT(self, timedelta_val): diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index 4eebb6d..4ab5964 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -665,6 +665,80 @@ def test_list_tables_failure_name_bad_before(self): self._list_tables_helper(table_id, table_name=bad_table_name) +class Test__pb_timestamp_to_datetime(unittest2.TestCase): + + def _callFUT(self, timestamp): + from gcloud_bigtable.cluster import _pb_timestamp_to_datetime + return _pb_timestamp_to_datetime(timestamp) + + def test_it(self): + import datetime + import pytz + from gcloud_bigtable._generated.timestamp_pb2 import Timestamp + + # Epoch is midnight on January 1, 1970 ... + dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0, + minute=1, second=1, microsecond=1234, + tzinfo=pytz.utc) + # ... so 1 minute and 1 second after is 61 seconds and 1234 + # microseconds is 1234000 nanoseconds. + timestamp = Timestamp(seconds=61, nanos=1234000) + self.assertEqual(self._callFUT(timestamp), dt_stamp) + + +class Test__parse_pb_any_to_native(unittest2.TestCase): + + def _callFUT(self, any_val, expected_type=None): + from gcloud_bigtable.cluster import _parse_pb_any_to_native + return _parse_pb_any_to_native(any_val, expected_type=expected_type) + + def test_it(self): + from gcloud_bigtable._generated import any_pb2 + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import cluster as MUT + + type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name + fake_type_url_map = {type_url: data_pb2.Cell} + + cell = data_pb2.Cell( + timestamp_micros=0, + value=b'foobar', + ) + any_val = any_pb2.Any( + type_url=type_url, + value=cell.SerializeToString(), + ) + with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): + result = self._callFUT(any_val) + + self.assertEqual(result, cell) + + def test_unknown_type_url(self): + from gcloud_bigtable._generated import any_pb2 + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import cluster as MUT + + fake_type_url_map = {} + any_val = any_pb2.Any() + with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): + with self.assertRaises(KeyError): + self._callFUT(any_val) + + def test_disagreeing_type_url(self): + from gcloud_bigtable._generated import any_pb2 + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import cluster as MUT + + type_url1 = 'foo' + type_url2 = 'bar' + fake_type_url_map = {type_url1: None} + any_val = any_pb2.Any(type_url=type_url2) + with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): + with self.assertRaises(ValueError): + self._callFUT(any_val, expected_type=type_url1) + + class _Client(object): cluster_stub = None From 6611e1bdfe0326bd7c448832484dfa5f544506d0 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 17:31:20 -0800 Subject: [PATCH 32/65] Bringing local _helpers.py closer to upstream. This is so that a simple diff is possible. This turned out to require quite a lot of changes! --- _fake_grpc/grpc/beta/interfaces.py | 13 + .../grpc/framework/interfaces/face/face.py | 13 + gcloud_bigtable/_helpers.py | 166 +------------ gcloud_bigtable/_non_upstream_helpers.py | 78 ++++++ gcloud_bigtable/cluster.py | 2 +- gcloud_bigtable/column_family.py | 51 +++- gcloud_bigtable/happybase/batch.py | 2 +- gcloud_bigtable/happybase/table.py | 6 +- gcloud_bigtable/happybase/test_batch.py | 3 +- gcloud_bigtable/happybase/test_table.py | 9 +- gcloud_bigtable/row.py | 4 +- gcloud_bigtable/row_data.py | 4 +- gcloud_bigtable/table.py | 2 +- gcloud_bigtable/test__helpers.py | 235 +----------------- gcloud_bigtable/test__non_upstream_helpers.py | 80 ++++++ gcloud_bigtable/test_column_family.py | 65 +++++ gcloud_bigtable/test_row.py | 9 +- gcloud_bigtable/test_row_data.py | 2 +- pylintrc_default | 2 +- system_tests/run.py | 13 +- 20 files changed, 349 insertions(+), 410 deletions(-) create mode 100644 _fake_grpc/grpc/beta/interfaces.py create mode 100644 _fake_grpc/grpc/framework/interfaces/face/face.py diff --git a/_fake_grpc/grpc/beta/interfaces.py b/_fake_grpc/grpc/beta/interfaces.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/beta/interfaces.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/face.py b/_fake_grpc/grpc/framework/interfaces/face/face.py new file mode 100644 index 0000000..61e4b79 --- /dev/null +++ b/_fake_grpc/grpc/framework/interfaces/face/face.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index e971b1e..c19371e 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -19,18 +19,13 @@ """ -import datetime import platform -import pytz -import six - from grpc.beta import implementations -from gcloud_bigtable._generated import duration_pb2 +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) # See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 # for appropriate values on other systems. _PLAT_SYS = platform.system() @@ -62,131 +57,23 @@ def __call__(self, ignored_val): ] -class AuthInfo(object): - """Local namespace for caching auth information.""" - - ROOT_CERTIFICATES = None - - -def _timedelta_to_duration_pb(timedelta_val): - """Convert a Python timedelta object to a duration protobuf. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type timedelta_val: :class:`datetime.timedelta` - :param timedelta_val: A timedelta object. - - :rtype: :class:`duration_pb2.Duration` - :returns: A duration object equivalent to the time delta. - """ - seconds_decimal = timedelta_val.total_seconds() - # Truncate the parts other than the integer. - seconds = int(seconds_decimal) - if seconds_decimal < 0: - signed_micros = timedelta_val.microseconds - 10**6 - else: - signed_micros = timedelta_val.microseconds - # Convert nanoseconds to microseconds. - nanos = 1000 * signed_micros - return duration_pb2.Duration(seconds=seconds, nanos=nanos) - - -def _duration_pb_to_timedelta(duration_pb): - """Convert a duration protobuf to a Python timedelta object. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type duration_pb: :class:`duration_pb2.Duration` - :param duration_pb: A protobuf duration object. - - :rtype: :class:`datetime.timedelta` - :returns: The converted timedelta object. - """ - return datetime.timedelta( - seconds=duration_pb.seconds, - microseconds=(duration_pb.nanos / 1000.0), - ) - - -def _timestamp_to_microseconds(timestamp, granularity=1000): - """Converts a native datetime object to microseconds. +def get_certs(): + """Gets the root certificates. .. note:: - If ``timestamp`` does not have the same timezone as ``EPOCH`` - (which is UTC), then subtracting the epoch from the timestamp - will raise a :class:`TypeError `. - - :type timestamp: :class:`datetime.datetime` - :param timestamp: A timestamp to be converted to microseconds. - - :type granularity: int - :param granularity: The resolution (relative to microseconds) that the - timestamp should be truncated to. Defaults to 1000 - and no other value is likely needed since the only - value of the enum - :class:`.data_pb2.Table.TimestampGranularity` is - :data:`.data_pb2.Table.MILLIS`. - - :rtype: int - :returns: The ``timestamp`` as microseconds (with the appropriate - granularity). - """ - timestamp_seconds = (timestamp - EPOCH).total_seconds() - timestamp_micros = int(10**6 * timestamp_seconds) - # Truncate to granularity. - timestamp_micros -= (timestamp_micros % granularity) - return timestamp_micros + This is only called by :func:`make_stub`. For most applications, + a few gRPC stubs (four total, one for each service) will be created + when a :class:`.Client` is created. This function will not likely + be used again while that application is running. - -def _microseconds_to_timestamp(microseconds): - """Converts microseconds to a native datetime object. - - :type microseconds: int - :param microseconds: The ``timestamp`` as microseconds. - - :rtype: :class:`datetime.datetime` - :returns: A timestamp to be converted from microseconds. - """ - return EPOCH + datetime.timedelta(microseconds=microseconds) - - -def _set_certs(): - """Sets the cached root certificates locally.""" - with open(SSL_CERT_FILE, mode='rb') as file_obj: - AuthInfo.ROOT_CERTIFICATES = file_obj.read() - - -def set_certs(reset=False): - """Sets the cached root certificates locally. - - If not manually told to reset or if the value is already set, - does nothing. - - :type reset: bool - :param reset: Boolean indicating if the cached certs should be reset. - """ - if AuthInfo.ROOT_CERTIFICATES is None or reset: - _set_certs() - - -def get_certs(): - """Gets the cached root certificates. - - Calls set_certs() first in case the value has not been set, but - this will do nothing if the value is already set. + However, it may be worthwhile to cache the output of this function. :rtype: str - :returns: The root certificates set on ``AuthInfo``. + :returns: The root certificates for the current machine. """ - set_certs(reset=False) - return AuthInfo.ROOT_CERTIFICATES + with open(SSL_CERT_FILE, mode='rb') as file_obj: + return file_obj.read() def make_stub(client, stub_factory, host, port): @@ -257,34 +144,3 @@ def _parse_family_pb(family_pb): cells.append(val_pair) return family_pb.name, result - - -def _to_bytes(value, encoding='ascii'): - """Converts a string value to bytes, if necessary. - - Unfortunately, ``six.b`` is insufficient for this task since in - Python2 it does not modify ``unicode`` objects. - - :type value: str / bytes or unicode - :param value: The string/bytes value to be converted. - - :type encoding: str - :param encoding: The encoding to use to convert unicode to bytes. Defaults - to "ascii", which will not allow any characters from - ordinals larger than 127. Other useful values are - "latin-1", which which will only allows byte ordinals - (up to 255) and "utf-8", which will encode any unicode - that needs to be. - - :rtype: str / bytes - :returns: The original value converted to bytes (if unicode) or as passed - in if it started out as bytes. - :raises: :class:`TypeError ` if the value - could not be converted to bytes. - """ - result = (value.encode(encoding) - if isinstance(value, six.text_type) else value) - if isinstance(result, six.binary_type): - return result - else: - raise TypeError('%r could not be converted to bytes' % (value,)) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 5fb4151..4c16709 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -18,9 +18,11 @@ """ +import datetime import os import socket +import pytz import six from oauth2client.client import GoogleCredentials @@ -36,6 +38,8 @@ PROJECT_ENV_VAR = 'GCLOUD_PROJECT' """Environment variable used to provide an implicit project ID.""" +EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) + def _project_from_environment(): """Attempts to get the project ID from an environment variable. @@ -213,3 +217,77 @@ def from_service_account_p12(cls, client_email, private_key_path, scope=None) kwargs['credentials'] = credentials return cls(*args, **kwargs) + + +def _timestamp_to_microseconds(timestamp, granularity=1000): + """Converts a native datetime object to microseconds. + + .. note:: + + If ``timestamp`` does not have the same timezone as ``EPOCH`` + (which is UTC), then subtracting the epoch from the timestamp + will raise a :class:`TypeError `. + + :type timestamp: :class:`datetime.datetime` + :param timestamp: A timestamp to be converted to microseconds. + + :type granularity: int + :param granularity: The resolution (relative to microseconds) that the + timestamp should be truncated to. Defaults to 1000 + and no other value is likely needed since the only + value of the enum + :class:`.data_pb2.Table.TimestampGranularity` is + :data:`.data_pb2.Table.MILLIS`. + + :rtype: int + :returns: The ``timestamp`` as microseconds (with the appropriate + granularity). + """ + timestamp_seconds = (timestamp - EPOCH).total_seconds() + timestamp_micros = int(10**6 * timestamp_seconds) + # Truncate to granularity. + timestamp_micros -= (timestamp_micros % granularity) + return timestamp_micros + + +def _microseconds_to_timestamp(microseconds): + """Converts microseconds to a native datetime object. + + :type microseconds: int + :param microseconds: The ``timestamp`` as microseconds. + + :rtype: :class:`datetime.datetime` + :returns: A timestamp to be converted from microseconds. + """ + return EPOCH + datetime.timedelta(microseconds=microseconds) + + +def _to_bytes(value, encoding='ascii'): + """Converts a string value to bytes, if necessary. + + Unfortunately, ``six.b`` is insufficient for this task since in + Python2 it does not modify ``unicode`` objects. + + :type value: str / bytes or unicode + :param value: The string/bytes value to be converted. + + :type encoding: str + :param encoding: The encoding to use to convert unicode to bytes. Defaults + to "ascii", which will not allow any characters from + ordinals larger than 127. Other useful values are + "latin-1", which which will only allows byte ordinals + (up to 255) and "utf-8", which will encode any unicode + that needs to be. + + :rtype: str / bytes + :returns: The original value converted to bytes (if unicode) or as passed + in if it started out as bytes. + :raises: :class:`TypeError ` if the value + could not be converted to bytes. + """ + result = (value.encode(encoding) + if isinstance(value, six.text_type) else value) + if isinstance(result, six.binary_type): + return result + else: + raise TypeError('%r could not be converted to bytes' % (value,)) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 98b2b84..61a99fc 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -24,7 +24,7 @@ from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import EPOCH as _EPOCH +from gcloud_bigtable._non_upstream_helpers import EPOCH as _EPOCH from gcloud_bigtable.table import Table diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index 1c5bb60..0b8687c 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -15,11 +15,58 @@ """User friendly container for Google Cloud Bigtable Column Family.""" +import datetime + from gcloud_bigtable._generated import bigtable_table_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _duration_pb_to_timedelta -from gcloud_bigtable._helpers import _timedelta_to_duration_pb +from gcloud_bigtable._generated import duration_pb2 + + +def _timedelta_to_duration_pb(timedelta_val): + """Convert a Python timedelta object to a duration protobuf. + + .. note:: + + The Python timedelta has a granularity of microseconds while + the protobuf duration type has a duration of nanoseconds. + + :type timedelta_val: :class:`datetime.timedelta` + :param timedelta_val: A timedelta object. + + :rtype: :class:`duration_pb2.Duration` + :returns: A duration object equivalent to the time delta. + """ + seconds_decimal = timedelta_val.total_seconds() + # Truncate the parts other than the integer. + seconds = int(seconds_decimal) + if seconds_decimal < 0: + signed_micros = timedelta_val.microseconds - 10**6 + else: + signed_micros = timedelta_val.microseconds + # Convert nanoseconds to microseconds. + nanos = 1000 * signed_micros + return duration_pb2.Duration(seconds=seconds, nanos=nanos) + + +def _duration_pb_to_timedelta(duration_pb): + """Convert a duration protobuf to a Python timedelta object. + + .. note:: + + The Python timedelta has a granularity of microseconds while + the protobuf duration type has a duration of nanoseconds. + + :type duration_pb: :class:`duration_pb2.Duration` + :param duration_pb: A protobuf duration object. + + :rtype: :class:`datetime.timedelta` + :returns: The converted timedelta object. + """ + return datetime.timedelta( + seconds=duration_pb.seconds, + microseconds=(duration_pb.nanos / 1000.0), + ) class GarbageCollectionRule(object): diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index 3b006ba..0fbdc65 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -18,7 +18,7 @@ import datetime import six -from gcloud_bigtable._helpers import _microseconds_to_timestamp +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable.row import TimestampRange diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index c7daf1b..be6bf1c 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -19,9 +19,9 @@ import six -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _timestamp_to_microseconds -from gcloud_bigtable._helpers import _to_bytes +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp +from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds +from gcloud_bigtable._non_upstream_helpers import _to_bytes from gcloud_bigtable.column_family import GarbageCollectionRule from gcloud_bigtable.column_family import GarbageCollectionRuleIntersection from gcloud_bigtable.happybase.batch import Batch diff --git a/gcloud_bigtable/happybase/test_batch.py b/gcloud_bigtable/happybase/test_batch.py index d3e7563..7ccee33 100644 --- a/gcloud_bigtable/happybase/test_batch.py +++ b/gcloud_bigtable/happybase/test_batch.py @@ -82,7 +82,8 @@ def test_constructor_defaults(self): self.assertEqual(batch._transaction, False) def test_constructor_explicit(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) from gcloud_bigtable.row import TimestampRange table = object() diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py index 94eb784..a5bfa85 100644 --- a/gcloud_bigtable/happybase/test_table.py +++ b/gcloud_bigtable/happybase/test_table.py @@ -147,7 +147,8 @@ def test_invalid_type(self): self._callFUT(timestamp=timestamp) def test_success(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) from gcloud_bigtable.row import TimestampRange timestamp = 1441928298571 @@ -176,7 +177,8 @@ def test_without_timestamp(self): self.assertEqual(result, [value1, value2]) def test_with_timestamp(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) from gcloud_bigtable.row_data import Cell value1 = 'foo' @@ -268,7 +270,8 @@ def test_with_versions(self): versions) def test_with_timestamp(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) from gcloud_bigtable.row import RowFilter from gcloud_bigtable.row import TimestampRange diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 917fd0e..044082d 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -23,8 +23,8 @@ from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud_bigtable._helpers import _parse_family_pb -from gcloud_bigtable._helpers import _timestamp_to_microseconds -from gcloud_bigtable._helpers import _to_bytes +from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds +from gcloud_bigtable._non_upstream_helpers import _to_bytes _MAX_MUTATIONS = 100000 diff --git a/gcloud_bigtable/row_data.py b/gcloud_bigtable/row_data.py index 3f9a80f..b8cab89 100644 --- a/gcloud_bigtable/row_data.py +++ b/gcloud_bigtable/row_data.py @@ -18,8 +18,8 @@ import copy import six -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _to_bytes +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp +from gcloud_bigtable._non_upstream_helpers import _to_bytes class Cell(object): diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index 27012fa..2d97729 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -20,7 +20,7 @@ bigtable_service_messages_pb2 as data_messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _to_bytes +from gcloud_bigtable._non_upstream_helpers import _to_bytes from gcloud_bigtable.column_family import ColumnFamily from gcloud_bigtable.column_family import _gc_rule_from_pb from gcloud_bigtable.row import Row diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 7fe0377..c9ef90a 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -73,220 +73,27 @@ class _ReturnVal(object): self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) -class Test__timedelta_to_duration_pb(unittest2.TestCase): - - def _callFUT(self, timedelta_val): - from gcloud_bigtable._helpers import _timedelta_to_duration_pb - return _timedelta_to_duration_pb(timedelta_val) - - def test_it(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = microseconds = 1 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds) - self.assertEqual(result.nanos, 1000 * microseconds) - - def test_with_negative_microseconds(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = 1 - microseconds = -5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds - 1) - self.assertEqual(result.nanos, 10**9 + 1000 * microseconds) - - def test_with_negative_seconds(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = -1 - microseconds = 5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds + 1) - self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds)) - - -class Test__duration_pb_to_timedelta(unittest2.TestCase): - - def _callFUT(self, duration_pb): - from gcloud_bigtable._helpers import _duration_pb_to_timedelta - return _duration_pb_to_timedelta(duration_pb) - - def test_it(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = microseconds = 1 - duration_pb = duration_pb2.Duration(seconds=seconds, - nanos=1000 * microseconds) - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(duration_pb) - self.assertTrue(isinstance(result, datetime.timedelta)) - self.assertEqual(result, timedelta_val) - - -class Test__timestamp_to_microseconds(unittest2.TestCase): - - def _callFUT(self, timestamp, granularity=1000): - from gcloud_bigtable._helpers import _timestamp_to_microseconds - return _timestamp_to_microseconds(timestamp, granularity=granularity) - - def test_default_granularity(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 898294371 - millis_granularity = microseconds - (microseconds % 1000) - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(millis_granularity, self._callFUT(timestamp)) - - def test_no_granularity(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 11122205067 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(microseconds, self._callFUT(timestamp, granularity=1)) - - def test_non_utc_timestamp(self): - import datetime - - epoch_no_tz = datetime.datetime.utcfromtimestamp(0) - with self.assertRaises(TypeError): - self._callFUT(epoch_no_tz) - - def test_non_datetime_timestamp(self): - timestamp = object() # Not a datetime object. - with self.assertRaises(TypeError): - self._callFUT(timestamp) - - -class Test__microseconds_to_timestamp(unittest2.TestCase): - - def _callFUT(self, microseconds): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - return _microseconds_to_timestamp(microseconds) - - def test_it(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 123456 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(timestamp, self._callFUT(microseconds)) - - -class Test__set_certs(unittest2.TestCase): +class Test_get_certs(unittest2.TestCase): def _callFUT(self): - from gcloud_bigtable._helpers import _set_certs - return _set_certs() + from gcloud_bigtable._helpers import get_certs + return get_certs() def test_it(self): import tempfile from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import _helpers as MUT - self.assertTrue(MUT.AuthInfo.ROOT_CERTIFICATES is None) - + # Just write to a mock file. filename = tempfile.mktemp() contents = b'FOOBARBAZ' with open(filename, 'wb') as file_obj: file_obj.write(contents) - with _Monkey(MUT, SSL_CERT_FILE=filename): - self._callFUT() - - self.assertEqual(MUT.AuthInfo.ROOT_CERTIFICATES, contents) - # Reset to `None` value checked above. - MUT.AuthInfo.ROOT_CERTIFICATES = None - - -class Test_set_certs(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._helpers import set_certs - return set_certs() - - def test_call_private(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_count = [0] - - def mock_set_certs(): - call_count[0] += 1 - - class _AuthInfo(object): - ROOT_CERTIFICATES = None - - with _Monkey(MUT, AuthInfo=_AuthInfo, - _set_certs=mock_set_certs): - self._callFUT() - - self.assertEqual(call_count, [1]) - - def test_do_nothing(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_count = [0] - - def mock_set_certs(): - call_count[0] += 1 - - # Make sure the fake method gets called by **someone** to make - # tox -e cover happy. - mock_set_certs() - self.assertEqual(call_count, [1]) - - class _AuthInfo(object): - ROOT_CERTIFICATES = object() - - with _Monkey(MUT, AuthInfo=_AuthInfo, - _set_certs=mock_set_certs): - self._callFUT() - - self.assertEqual(call_count, [1]) - - -class Test_get_certs(unittest2.TestCase): - def _callFUT(self): - from gcloud_bigtable._helpers import get_certs - return get_certs() - - def test_it(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_kwargs = [] - return_val = object() - - def mock_set_certs(**kwargs): - call_kwargs.append(kwargs) - - class _AuthInfo(object): - ROOT_CERTIFICATES = return_val - - with _Monkey(MUT, AuthInfo=_AuthInfo, - set_certs=mock_set_certs): + with _Monkey(MUT, SSL_CERT_FILE=filename): result = self._callFUT() - self.assertEqual(call_kwargs, [{'reset': False}]) - self.assertTrue(result is return_val) + self.assertEqual(result, contents) class Test_make_stub(unittest2.TestCase): @@ -345,7 +152,8 @@ def _callFUT(self, family_pb): def test_it(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) COL_FAM1 = u'col-fam-id' COL_NAME1 = b'col-name1' @@ -394,30 +202,3 @@ def test_it(self): ], ) self.assertEqual(expected_output, self._callFUT(sample_input)) - - -class Test__to_bytes(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable._helpers import _to_bytes - return _to_bytes(*args, **kwargs) - - def test_with_bytes(self): - value = b'bytes-val' - self.assertEqual(self._callFUT(value), value) - - def test_with_unicode(self): - value = u'string-val' - encoded_value = b'string-val' - self.assertEqual(self._callFUT(value), encoded_value) - - def test_unicode_non_ascii(self): - value = u'\u2013' # Long hyphen - encoded_value = b'\xe2\x80\x93' - self.assertRaises(UnicodeEncodeError, self._callFUT, value) - self.assertEqual(self._callFUT(value, encoding='utf-8'), - encoded_value) - - def test_with_nonstring_type(self): - value = object() - self.assertRaises(TypeError, self._callFUT, value) diff --git a/gcloud_bigtable/test__non_upstream_helpers.py b/gcloud_bigtable/test__non_upstream_helpers.py index 426e3d0..5fdaad1 100644 --- a/gcloud_bigtable/test__non_upstream_helpers.py +++ b/gcloud_bigtable/test__non_upstream_helpers.py @@ -305,3 +305,83 @@ def test_from_service_account_p12(self): signed_creds.check_called(self, [()], [signed_creds_kw]) # Load private key (via _get_contents) from the key path. mock_get_contents.check_called(self, [(private_key_path,)]) + + +class Test__timestamp_to_microseconds(unittest2.TestCase): + + def _callFUT(self, timestamp, granularity=1000): + from gcloud_bigtable._non_upstream_helpers import ( + _timestamp_to_microseconds) + return _timestamp_to_microseconds(timestamp, granularity=granularity) + + def test_default_granularity(self): + import datetime + from gcloud_bigtable import _non_upstream_helpers as MUT + + microseconds = 898294371 + millis_granularity = microseconds - (microseconds % 1000) + timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) + self.assertEqual(millis_granularity, self._callFUT(timestamp)) + + def test_no_granularity(self): + import datetime + from gcloud_bigtable import _non_upstream_helpers as MUT + + microseconds = 11122205067 + timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) + self.assertEqual(microseconds, self._callFUT(timestamp, granularity=1)) + + def test_non_utc_timestamp(self): + import datetime + + epoch_no_tz = datetime.datetime.utcfromtimestamp(0) + with self.assertRaises(TypeError): + self._callFUT(epoch_no_tz) + + def test_non_datetime_timestamp(self): + timestamp = object() # Not a datetime object. + with self.assertRaises(TypeError): + self._callFUT(timestamp) + + +class Test__microseconds_to_timestamp(unittest2.TestCase): + + def _callFUT(self, microseconds): + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) + return _microseconds_to_timestamp(microseconds) + + def test_it(self): + import datetime + from gcloud_bigtable import _non_upstream_helpers as MUT + + microseconds = 123456 + timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) + self.assertEqual(timestamp, self._callFUT(microseconds)) + + +class Test__to_bytes(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud_bigtable._non_upstream_helpers import _to_bytes + return _to_bytes(*args, **kwargs) + + def test_with_bytes(self): + value = b'bytes-val' + self.assertEqual(self._callFUT(value), value) + + def test_with_unicode(self): + value = u'string-val' + encoded_value = b'string-val' + self.assertEqual(self._callFUT(value), encoded_value) + + def test_unicode_non_ascii(self): + value = u'\u2013' # Long hyphen + encoded_value = b'\xe2\x80\x93' + self.assertRaises(UnicodeEncodeError, self._callFUT, value) + self.assertEqual(self._callFUT(value, encoding='utf-8'), + encoded_value) + + def test_with_nonstring_type(self): + value = object() + self.assertRaises(TypeError, self._callFUT, value) diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py index e2ae9f4..9a685ea 100644 --- a/gcloud_bigtable/test_column_family.py +++ b/gcloud_bigtable/test_column_family.py @@ -568,6 +568,71 @@ def test_unknown_field_name(self): self.assertEqual(self._callFUT(gc_rule_pb), None) +class Test__timedelta_to_duration_pb(unittest2.TestCase): + + def _callFUT(self, timedelta_val): + from gcloud_bigtable.column_family import _timedelta_to_duration_pb + return _timedelta_to_duration_pb(timedelta_val) + + def test_it(self): + import datetime + from gcloud_bigtable._generated import duration_pb2 + + seconds = microseconds = 1 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds) + self.assertEqual(result.nanos, 1000 * microseconds) + + def test_with_negative_microseconds(self): + import datetime + from gcloud_bigtable._generated import duration_pb2 + + seconds = 1 + microseconds = -5 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds - 1) + self.assertEqual(result.nanos, 10**9 + 1000 * microseconds) + + def test_with_negative_seconds(self): + import datetime + from gcloud_bigtable._generated import duration_pb2 + + seconds = -1 + microseconds = 5 + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(timedelta_val) + self.assertTrue(isinstance(result, duration_pb2.Duration)) + self.assertEqual(result.seconds, seconds + 1) + self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds)) + + +class Test__duration_pb_to_timedelta(unittest2.TestCase): + + def _callFUT(self, duration_pb): + from gcloud_bigtable.column_family import _duration_pb_to_timedelta + return _duration_pb_to_timedelta(duration_pb) + + def test_it(self): + import datetime + from gcloud_bigtable._generated import duration_pb2 + + seconds = microseconds = 1 + duration_pb = duration_pb2.Duration(seconds=seconds, + nanos=1000 * microseconds) + timedelta_val = datetime.timedelta(seconds=seconds, + microseconds=microseconds) + result = self._callFUT(duration_pb) + self.assertTrue(isinstance(result, datetime.timedelta)) + self.assertEqual(result, timedelta_val) + + class _Client(object): cluster_stub = None diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index c965c23..d3cbd75 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -210,7 +210,7 @@ def test_set_cell_with_non_bytes_value(self): def test_set_cell_with_non_null_timestamp(self): import datetime - from gcloud_bigtable. _helpers import EPOCH + from gcloud_bigtable._non_upstream_helpers import EPOCH microseconds = 898294371 millis_granularity = microseconds - (microseconds % 1000) @@ -318,7 +318,7 @@ def test_delete_cells_no_time_range(self): def test_delete_cells_with_time_range(self): import datetime - from gcloud_bigtable. _helpers import EPOCH + from gcloud_bigtable._non_upstream_helpers import EPOCH from gcloud_bigtable.row import TimestampRange microseconds = 30871000 # Makes sure already milliseconds granularity @@ -614,7 +614,8 @@ def _callFUT(self, row_response): def test_it(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import _microseconds_to_timestamp + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) COL_FAM1 = u'col-fam-id' COL_FAM2 = u'col-fam-id2' @@ -1002,7 +1003,7 @@ def test___ne__same_value(self): def _to_pb_helper(self, start_micros=None, end_micros=None): import datetime from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable. _helpers import EPOCH + from gcloud_bigtable._non_upstream_helpers import EPOCH pb_kwargs = {} diff --git a/gcloud_bigtable/test_row_data.py b/gcloud_bigtable/test_row_data.py index 5a29433..ff86ee9 100644 --- a/gcloud_bigtable/test_row_data.py +++ b/gcloud_bigtable/test_row_data.py @@ -28,7 +28,7 @@ def _makeOne(self, *args, **kwargs): def _from_pb_test_helper(self, labels=None): import datetime from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import EPOCH + from gcloud_bigtable._non_upstream_helpers import EPOCH timestamp_micros = 18738724000 # Make sure millis granularity timestamp = EPOCH + datetime.timedelta(microseconds=timestamp_micros) diff --git a/pylintrc_default b/pylintrc_default index 6e33cbc..79108b1 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -205,7 +205,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1545 +max-module-lines=1546 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). diff --git a/system_tests/run.py b/system_tests/run.py index f90253d..8945d91 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -24,8 +24,8 @@ from oauth2client.client import GoogleCredentials -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _timestamp_to_microseconds +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp +from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable.client import Client from gcloud_bigtable.column_family import GarbageCollectionRule from gcloud_bigtable.row import RowFilter @@ -204,8 +204,8 @@ def test_create_table(self): self.assertEqual(sorted_tables, expected_tables) def test_rename_table(self): - from grpc.beta.interfaces import StatusCode - from grpc.framework.interfaces.face.face import LocalError + from grpc.beta import interfaces + from grpc.framework.interfaces.face import face temp_table_id = 'foo-bar-baz-table' temp_table = CLUSTER.table(temp_table_id) @@ -215,10 +215,11 @@ def test_rename_table(self): exc_caught = None try: temp_table.rename(temp_table_id + '-alt') - except LocalError as exc: + except face.LocalError as exc: exc_caught = exc # Python 3 syntax issue. self.assertNotEqual(exc_caught, None) - self.assertEqual(exc_caught.code, StatusCode.UNIMPLEMENTED) + self.assertEqual(exc_caught.code, + interfaces.StatusCode.UNIMPLEMENTED) self.assertEqual( exc_caught.details, 'BigtableTableService.RenameTable is not yet implemented') From 4f0ee5e18d0fe4bd01232d51a36587e02a99918a Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 17:35:17 -0800 Subject: [PATCH 33/65] Bringing local table.py closer to upstream. This is so that a simple diff is possible. --- gcloud_bigtable/table.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index 2d97729..c604c4a 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -86,9 +86,7 @@ def column_family(self, column_family_id, gc_rule=None): :param column_family_id: The ID of the column family. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - :type gc_rule: :class:`.column_family.GarbageCollectionRule`, - :class:`.column_family.GarbageCollectionRuleUnion` or - :class:`.column_family.GarbageCollectionRuleIntersection` + :type gc_rule: :class:`.column_family.GarbageCollectionRule` :param gc_rule: (Optional) The garbage collection settings for this column family. @@ -103,10 +101,7 @@ def row(self, row_key, filter_=None): :type row_key: bytes :param row_key: The key for the row being created. - :type filter_: :class:`.RowFilter`, - :class:`.RowFilterChain`, - :class:`.RowFilterUnion`, or - :class:`.ConditionalRowFilter` + :type filter_: :class:`.RowFilter` :param filter_: (Optional) Filter to be used for conditional mutations. See :class:`.Row` for more details. @@ -199,7 +194,7 @@ def delete(self): client._table_stub.DeleteTable(request_pb, client.timeout_seconds) def list_column_families(self): - """Check if this table exists. + """List the column families owned by this table. :rtype: dictionary with string as keys and :class:`.column_family.ColumnFamily` as values From 4f3132ee3df6d21a2b7ab1a7b55501e8caad41ab Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 18:36:56 -0800 Subject: [PATCH 34/65] Bringing local column_family.py closer to upstream. This is so that a simple diff is possible. --- gcloud_bigtable/_non_upstream_helpers.py | 31 ++ gcloud_bigtable/column_family.py | 237 ++++++-------- gcloud_bigtable/happybase/connection.py | 26 +- gcloud_bigtable/happybase/table.py | 41 ++- gcloud_bigtable/happybase/test_connection.py | 21 +- gcloud_bigtable/happybase/test_table.py | 49 ++- gcloud_bigtable/test__non_upstream_helpers.py | 29 ++ gcloud_bigtable/test_column_family.py | 288 ++++++++---------- 8 files changed, 338 insertions(+), 384 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 4c16709..78c994a 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -21,6 +21,7 @@ import datetime import os import socket +import sys import pytz import six @@ -291,3 +292,33 @@ def _to_bytes(value, encoding='ascii'): return result else: raise TypeError('%r could not be converted to bytes' % (value,)) + + +def _total_seconds_backport(offset): + """Backport of timedelta.total_seconds() from python 2.7+. + + :type offset: :class:`datetime.timedelta` + :param offset: A timedelta object. + + :rtype: int + :returns: The total seconds (including microseconds) in the + duration. + """ + seconds = offset.days * 24 * 60 * 60 + offset.seconds + return seconds + offset.microseconds * 1e-6 + + +def _total_seconds(offset): + """Version independent total seconds for a time delta. + + :type offset: :class:`datetime.timedelta` + :param offset: A timedelta object. + + :rtype: int + :returns: The total seconds (including microseconds) in the + duration. + """ + if sys.version_info[:2] < (2, 7): # pragma: NO COVER + return _total_seconds_backport(offset) + else: + return offset.total_seconds() diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index 0b8687c..eabdd50 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -21,6 +21,7 @@ from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import duration_pb2 +from gcloud_bigtable._non_upstream_helpers import _total_seconds def _timedelta_to_duration_pb(timedelta_val): @@ -34,10 +35,10 @@ def _timedelta_to_duration_pb(timedelta_val): :type timedelta_val: :class:`datetime.timedelta` :param timedelta_val: A timedelta object. - :rtype: :class:`duration_pb2.Duration` + :rtype: :class:`.duration_pb2.Duration` :returns: A duration object equivalent to the time delta. """ - seconds_decimal = timedelta_val.total_seconds() + seconds_decimal = _total_seconds(timedelta_val) # Truncate the parts other than the integer. seconds = int(seconds_decimal) if seconds_decimal < 0: @@ -57,7 +58,7 @@ def _duration_pb_to_timedelta(duration_pb): The Python timedelta has a granularity of microseconds while the protobuf duration type has a duration of nanoseconds. - :type duration_pb: :class:`duration_pb2.Duration` + :type duration_pb: :class:`.duration_pb2.Duration` :param duration_pb: A protobuf duration object. :rtype: :class:`datetime.timedelta` @@ -70,85 +71,83 @@ def _duration_pb_to_timedelta(duration_pb): class GarbageCollectionRule(object): - """Table garbage collection rule. + """Garbage collection rule for column families within a table. - Cells in the table fitting the rule will be deleted during - garbage collection. - - These values can be combined via :class:`GarbageCollectionRuleUnion` and - :class:`GarbageCollectionRuleIntersection`. + Cells in the column family (within a table) fitting the rule will be + deleted during garbage collection. .. note:: - At most one of ``max_num_versions`` and ``max_age`` can be specified - at once. + This class is a do-nothing base class for all GC rules. .. note:: A string ``gc_expression`` can also be used with API requests, but that value would be superceded by a ``gc_rule``. As a result, we - don't support that feature and instead support via this native - object. + don't support that feature and instead support via native classes. + """ - :type max_num_versions: int - :param max_num_versions: The maximum number of versions + def __ne__(self, other): + return not self.__eq__(other) - :type max_age: :class:`datetime.timedelta` - :param max_age: The maximum age allowed for a cell in the table. - :raises: :class:`TypeError ` if both - ``max_num_versions`` and ``max_age`` are set. +class MaxVersionsGCRule(GarbageCollectionRule): + """Garbage collection limiting the number of versions of a cell. + + :type max_num_versions: int + :param max_num_versions: The maximum number of versions """ - def __init__(self, max_num_versions=None, max_age=None): + def __init__(self, max_num_versions): self.max_num_versions = max_num_versions - self.max_age = max_age - self._check_single_value() - def _check_single_value(self): - """Checks that at most one value is set on the instance. + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.max_num_versions == self.max_num_versions + + def to_pb(self): + """Converts the garbage collection rule to a protobuf. - :raises: :class:`TypeError ` if not exactly one - value set on the instance. + :rtype: :class:`.data_pb2.GcRule` + :returns: The converted current object. """ - if self.max_num_versions is not None and self.max_age is not None: - raise TypeError('At most one of max_num_versions and ' - 'max_age can be set') + return data_pb2.GcRule(max_num_versions=self.max_num_versions) + + +class MaxAgeGCRule(GarbageCollectionRule): + """Garbage collection limiting the age of a cell. + + :type max_age: :class:`datetime.timedelta` + :param max_age: The maximum age allowed for a cell in the table. + """ + + def __init__(self, max_age): + self.max_age = max_age def __eq__(self, other): if not isinstance(other, self.__class__): return False - return (other.max_num_versions == self.max_num_versions and - other.max_age == self.max_age) - - def __ne__(self, other): - return not self.__eq__(other) + return other.max_age == self.max_age def to_pb(self): - """Converts the :class:`GarbageCollectionRule` to a protobuf. + """Converts the garbage collection rule to a protobuf. :rtype: :class:`.data_pb2.GcRule` :returns: The converted current object. """ - self._check_single_value() - gc_rule_kwargs = {} - if self.max_num_versions is not None: - gc_rule_kwargs['max_num_versions'] = self.max_num_versions - if self.max_age is not None: - gc_rule_kwargs['max_age'] = _timedelta_to_duration_pb(self.max_age) - return data_pb2.GcRule(**gc_rule_kwargs) + max_age = _timedelta_to_duration_pb(self.max_age) + return data_pb2.GcRule(max_age=max_age) -class GarbageCollectionRuleUnion(object): +class GCRuleUnion(GarbageCollectionRule): """Union of garbage collection rules. :type rules: list - :param rules: List of :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` and/or - :class:`GarbageCollectionRuleIntersection` + :param rules: List of :class:`GarbageCollectionRule`. """ - def __init__(self, rules=None): + def __init__(self, rules): self.rules = rules def __eq__(self, other): @@ -156,9 +155,6 @@ def __eq__(self, other): return False return other.rules == self.rules - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): """Converts the union into a single gc rule as a protobuf. @@ -170,16 +166,14 @@ def to_pb(self): return data_pb2.GcRule(union=union) -class GarbageCollectionRuleIntersection(object): +class GCRuleIntersection(GarbageCollectionRule): """Intersection of garbage collection rules. :type rules: list - :param rules: List of :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` and/or - :class:`GarbageCollectionRuleIntersection` + :param rules: List of :class:`GarbageCollectionRule`. """ - def __init__(self, rules=None): + def __init__(self, rules): self.rules = rules def __eq__(self, other): @@ -187,9 +181,6 @@ def __eq__(self, other): return False return other.rules == self.rules - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): """Converts the intersection into a single gc rule as a protobuf. @@ -217,9 +208,7 @@ class ColumnFamily(object): :type table: :class:`Table ` :param table: The table that owns the column family. - :type gc_rule: :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` or - :class:`GarbageCollectionRuleIntersection` + :type gc_rule: :class:`GarbageCollectionRule` :param gc_rule: (Optional) The garbage collection settings for this column family. """ @@ -229,33 +218,6 @@ def __init__(self, column_family_id, table, gc_rule=None): self._table = table self.gc_rule = gc_rule - @property - def table(self): - """Getter for column family's table. - - :rtype: :class:`Table ` - :returns: The table stored on the column family. - """ - return self._table - - @property - def client(self): - """Getter for column family's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this column family. - """ - return self.table._cluster._client - - @property - def timeout_seconds(self): - """Getter for column family's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default. - """ - return self.table._cluster._client.timeout_seconds - @property def name(self): """Column family name used in requests. @@ -272,111 +234,88 @@ def name(self): :rtype: str :returns: The column family name. """ - return self.table.name + '/columnFamilies/' + self.column_family_id + return self._table.name + '/columnFamilies/' + self.column_family_id def __eq__(self, other): if not isinstance(other, self.__class__): return False return (other.column_family_id == self.column_family_id and - other.gc_rule == self.gc_rule and - other.table == self.table) + other._table == self._table and + other.gc_rule == self.gc_rule) def __ne__(self, other): return not self.__eq__(other) - def create(self, timeout_seconds=None): - """Create this column family. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. - """ + def create(self): + """Create this column family.""" if self.gc_rule is None: column_family = data_pb2.ColumnFamily() else: column_family = data_pb2.ColumnFamily(gc_rule=self.gc_rule.to_pb()) request_pb = messages_pb2.CreateColumnFamilyRequest( - name=self.table.name, + name=self._table.name, column_family_id=self.column_family_id, column_family=column_family, ) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._table._cluster._client # We expect a `.data_pb2.ColumnFamily` - self.client._table_stub.CreateColumnFamily(request_pb, - timeout_seconds) + client._table_stub.CreateColumnFamily(request_pb, + client.timeout_seconds) - def update(self, timeout_seconds=None): + def update(self): """Update this column family. .. note:: Only the GC rule can be updated. By changing the column family ID, you will simply be referring to a different column family. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. """ request_kwargs = {'name': self.name} if self.gc_rule is not None: request_kwargs['gc_rule'] = self.gc_rule.to_pb() request_pb = data_pb2.ColumnFamily(**request_kwargs) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._table._cluster._client # We expect a `.data_pb2.ColumnFamily` - self.client._table_stub.UpdateColumnFamily(request_pb, - timeout_seconds) - - def delete(self, timeout_seconds=None): - """Delete this column family. + client._table_stub.UpdateColumnFamily(request_pb, + client.timeout_seconds) - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. - """ + def delete(self): + """Delete this column family.""" request_pb = messages_pb2.DeleteColumnFamilyRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds + client = self._table._cluster._client # We expect a `._generated.empty_pb2.Empty` - self.client._table_stub.DeleteColumnFamily(request_pb, - timeout_seconds) + client._table_stub.DeleteColumnFamily(request_pb, + client.timeout_seconds) def _gc_rule_from_pb(gc_rule_pb): - """Convert a protobuf GC rule to a Python version. + """Convert a protobuf GC rule to a native object. :type gc_rule_pb: :class:`.data_pb2.GcRule` :param gc_rule_pb: The GC rule to convert. - :rtype: :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion`, - :class:`GarbageCollectionRuleIntersection` or - :data:`NoneType ` + :rtype: :class:`GarbageCollectionRule` or :data:`NoneType ` :returns: An instance of one of the native rules defined in :module:`column_family` or :data:`None` if no values were set on the protobuf passed in. - :raises: :class:`ValueError ` if more than one - property has been set on the GC rule. + :raises: :class:`ValueError ` if the rule name + is unexpected. """ - all_fields = [field.name for field in gc_rule_pb._fields] - if len(all_fields) == 0: + rule_name = gc_rule_pb.WhichOneof('rule') + if rule_name is None: return None - elif len(all_fields) > 1: - raise ValueError('At most one field can be set on a GC rule.') - - field_name = all_fields[0] - if field_name == 'max_num_versions': - return GarbageCollectionRule( - max_num_versions=gc_rule_pb.max_num_versions) - elif field_name == 'max_age': + + if rule_name == 'max_num_versions': + return MaxVersionsGCRule(gc_rule_pb.max_num_versions) + elif rule_name == 'max_age': max_age = _duration_pb_to_timedelta(gc_rule_pb.max_age) - return GarbageCollectionRule(max_age=max_age) - elif field_name == 'union': - all_rules = gc_rule_pb.union.rules - return GarbageCollectionRuleUnion( - rules=[_gc_rule_from_pb(rule) for rule in all_rules]) - elif field_name == 'intersection': - all_rules = gc_rule_pb.intersection.rules - return GarbageCollectionRuleIntersection( - rules=[_gc_rule_from_pb(rule) for rule in all_rules]) + return MaxAgeGCRule(max_age) + elif rule_name == 'union': + return GCRuleUnion([_gc_rule_from_pb(rule) + for rule in gc_rule_pb.union.rules]) + elif rule_name == 'intersection': + rules = [_gc_rule_from_pb(rule) + for rule in gc_rule_pb.intersection.rules] + return GCRuleIntersection(rules) + else: + raise ValueError('Unexpected rule name', rule_name) diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index ddbfc39..45a15bd 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -19,8 +19,9 @@ import six from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import GarbageCollectionRule -from gcloud_bigtable.column_family import GarbageCollectionRuleIntersection +from gcloud_bigtable.column_family import GCRuleIntersection +from gcloud_bigtable.column_family import MaxAgeGCRule +from gcloud_bigtable.column_family import MaxVersionsGCRule from gcloud_bigtable.happybase.table import Table from gcloud_bigtable.table import Table as _LowLevelTable @@ -86,15 +87,11 @@ def _parse_family_option(option): :type option: :class:`dict`, :data:`NoneType `, - :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion`, - :class:`.GarbageCollectionRuleIntersection` + :class:`.GarbageCollectionRule` :param option: A column family option passes as a dictionary value in :meth:`Connection.create_table`. - :rtype: :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion`, - :class:`.GarbageCollectionRuleIntersection` + :rtype: :class:`.GarbageCollectionRule` :returns: A garbage collection rule parsed from the input. :raises: :class:`ValueError ` if ``option`` is a dictionary but keys other than ``max_versions`` and @@ -118,14 +115,13 @@ def _parse_family_option(option): result = None elif len(result) == 1: if max_num_versions is None: - result = GarbageCollectionRule(max_age=max_age) + result = MaxAgeGCRule(max_age) else: - result = GarbageCollectionRule( - max_num_versions=max_num_versions) + result = MaxVersionsGCRule(max_num_versions) else: # By our check above we know this means len(result) == 2. - rule1 = GarbageCollectionRule(max_age=max_age) - rule2 = GarbageCollectionRule(max_num_versions=max_num_versions) - result = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) + rule1 = MaxAgeGCRule(max_age) + rule2 = MaxVersionsGCRule(max_num_versions) + result = GCRuleIntersection(rules=[rule1, rule2]) return result @@ -364,8 +360,6 @@ def create_table(self, name, families): * :class:`dict` * :class:`.GarbageCollectionRule` - * :class:`.GarbageCollectionRuleUnion` - * :class:`.GarbageCollectionRuleIntersection` :raises: :class:`TypeError ` if ``families`` is not a dictionary, diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index be6bf1c..4f18470 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -22,8 +22,9 @@ from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable._non_upstream_helpers import _to_bytes -from gcloud_bigtable.column_family import GarbageCollectionRule -from gcloud_bigtable.column_family import GarbageCollectionRuleIntersection +from gcloud_bigtable.column_family import GCRuleIntersection +from gcloud_bigtable.column_family import MaxAgeGCRule +from gcloud_bigtable.column_family import MaxVersionsGCRule from gcloud_bigtable.happybase.batch import Batch from gcloud_bigtable.happybase.batch import _WAL_SENTINEL from gcloud_bigtable.happybase.batch import _get_column_pairs @@ -38,6 +39,7 @@ _DEFAULT_BATCH_SIZE = object() _DEFAULT_SCAN_BATCHING = object() _DEFAULT_SORTED_COLUMNS = object() +_SIMPLE_GC_RULES = (MaxAgeGCRule, MaxVersionsGCRule) def make_row(cell_map, include_timestamp): @@ -98,40 +100,33 @@ def _gc_rule_to_dict(gc_rule): Only does this if the garbage collection rule is: - * Simple :class:`.GarbageCollectionRule` with ``max_age`` - * Simple :class:`.GarbageCollectionRule` with ``max_num_versions`` - * Composite :class:`.GarbageCollectionRuleIntersection` with - two rules each for ``max_age`` and ``max_num_versions`` + * :class:`.MaxAgeGCRule` + * :class:`.MaxVersionsGCRule` + * Composite :class:`.GCRuleIntersection` with two rules, one each + of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` Otherwise, just returns the input without change. :type gc_rule: :data:`NoneType `, - :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleIntersection`, or - :class:`.GarbageCollectionRuleUnion` - :param gc_rule: A garbae collection rule to convert to a dictionary + :class:`.GarbageCollectionRule` + :param gc_rule: A garbage collection rule to convert to a dictionary (if possible). - :rtype: dict, - :class:`.GarbageCollectionRuleIntersection`, or - :class:`.GarbageCollectionRuleUnion` + :rtype: dict or :class:`.GarbageCollectionRule` :returns: The converted garbage collection rule. """ result = gc_rule if gc_rule is None: result = {} - elif isinstance(gc_rule, GarbageCollectionRule): - result = {} - # We assume that the GC rule has a single value. - if gc_rule.max_num_versions is not None: - result['max_versions'] = gc_rule.max_num_versions - if gc_rule.max_age is not None: - result['time_to_live'] = gc_rule.max_age.total_seconds() - elif isinstance(gc_rule, GarbageCollectionRuleIntersection): + elif isinstance(gc_rule, MaxAgeGCRule): + result = {'time_to_live': gc_rule.max_age.total_seconds()} + elif isinstance(gc_rule, MaxVersionsGCRule): + result = {'max_versions': gc_rule.max_num_versions} + elif isinstance(gc_rule, GCRuleIntersection): if len(gc_rule.rules) == 2: rule1, rule2 = gc_rule.rules - if (isinstance(rule1, GarbageCollectionRule) and - isinstance(rule2, GarbageCollectionRule)): + if (isinstance(rule1, _SIMPLE_GC_RULES) and + isinstance(rule2, _SIMPLE_GC_RULES)): rule1 = _gc_rule_to_dict(rule1) rule2 = _gc_rule_to_dict(rule2) key1, = rule1.keys() diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index d126bb7..ce5cf67 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -92,32 +92,32 @@ def test_dictionary_bad_key(self): self._callFUT(option) def test_dictionary_versions_key(self): - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxVersionsGCRule versions = 42 option = {'max_versions': versions} result = self._callFUT(option) - gc_rule = GarbageCollectionRule(max_num_versions=versions) + gc_rule = MaxVersionsGCRule(versions) self.assertEqual(result, gc_rule) def test_dictionary_ttl_key(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule time_to_live = 24 * 60 * 60 max_age = datetime.timedelta(days=1) option = {'time_to_live': time_to_live} result = self._callFUT(option) - gc_rule = GarbageCollectionRule(max_age=max_age) + gc_rule = MaxAgeGCRule(max_age) self.assertEqual(result, gc_rule) def test_dictionary_both_keys(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) + from gcloud_bigtable.column_family import GCRuleIntersection + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule versions = 42 time_to_live = 24 * 60 * 60 @@ -130,10 +130,9 @@ def test_dictionary_both_keys(self): max_age = datetime.timedelta(days=1) # NOTE: This relies on the order of the rules in the method we are # calling matching this order here. - gc_rule1 = GarbageCollectionRule(max_age=max_age) - gc_rule2 = GarbageCollectionRule(max_num_versions=versions) - gc_rule = GarbageCollectionRuleIntersection( - rules=[gc_rule1, gc_rule2]) + gc_rule1 = MaxAgeGCRule(max_age) + gc_rule2 = MaxVersionsGCRule(versions) + gc_rule = GCRuleIntersection(rules=[gc_rule1, gc_rule2]) self.assertEqual(result, gc_rule) def test_non_dictionary(self): diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py index a5bfa85..89a4921 100644 --- a/gcloud_bigtable/happybase/test_table.py +++ b/gcloud_bigtable/happybase/test_table.py @@ -50,21 +50,21 @@ def test_with_null(self): self.assertEqual(result, {}) def test_with_max_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxVersionsGCRule max_versions = 2 - gc_rule = GarbageCollectionRule(max_num_versions=max_versions) + gc_rule = MaxVersionsGCRule(max_versions) result = self._callFUT(gc_rule) expected_result = {'max_versions': max_versions} self.assertEqual(result, expected_result) def test_with_max_age(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule time_to_live = 101 max_age = datetime.timedelta(seconds=time_to_live) - gc_rule = GarbageCollectionRule(max_age=max_age) + gc_rule = MaxAgeGCRule(max_age) result = self._callFUT(gc_rule) expected_result = {'time_to_live': time_to_live} self.assertEqual(result, expected_result) @@ -75,43 +75,41 @@ def test_with_non_gc_rule(self): self.assertTrue(result is gc_rule) def test_with_gc_rule_union(self): - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion + from gcloud_bigtable.column_family import GCRuleUnion - gc_rule = GarbageCollectionRuleUnion() + gc_rule = GCRuleUnion(rules=[]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) def test_with_intersection_other_than_two(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) + from gcloud_bigtable.column_family import GCRuleIntersection - gc_rule = GarbageCollectionRuleIntersection(rules=[]) + gc_rule = GCRuleIntersection(rules=[]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) def test_with_intersection_two_max_num_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) + from gcloud_bigtable.column_family import GCRuleIntersection + from gcloud_bigtable.column_family import MaxVersionsGCRule - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule(max_num_versions=2) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) + rule1 = MaxVersionsGCRule(1) + rule2 = MaxVersionsGCRule(2) + gc_rule = GCRuleIntersection(rules=[rule1, rule2]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) def test_with_intersection_two_rules(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) + from gcloud_bigtable.column_family import GCRuleIntersection + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule time_to_live = 101 max_age = datetime.timedelta(seconds=time_to_live) - rule1 = GarbageCollectionRule(max_age=max_age) + rule1 = MaxAgeGCRule(max_age) max_versions = 2 - rule2 = GarbageCollectionRule(max_num_versions=max_versions) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) + rule2 = MaxVersionsGCRule(max_versions) + gc_rule = GCRuleIntersection(rules=[rule1, rule2]) result = self._callFUT(gc_rule) expected_result = { 'max_versions': max_versions, @@ -120,12 +118,11 @@ def test_with_intersection_two_rules(self): self.assertEqual(result, expected_result) def test_with_intersection_two_nested_rules(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) + from gcloud_bigtable.column_family import GCRuleIntersection - rule1 = GarbageCollectionRuleIntersection(rules=[]) - rule2 = GarbageCollectionRuleIntersection(rules=[]) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) + rule1 = GCRuleIntersection(rules=[]) + rule2 = GCRuleIntersection(rules=[]) + gc_rule = GCRuleIntersection(rules=[rule1, rule2]) result = self._callFUT(gc_rule) self.assertTrue(result is gc_rule) diff --git a/gcloud_bigtable/test__non_upstream_helpers.py b/gcloud_bigtable/test__non_upstream_helpers.py index 5fdaad1..31a19e8 100644 --- a/gcloud_bigtable/test__non_upstream_helpers.py +++ b/gcloud_bigtable/test__non_upstream_helpers.py @@ -385,3 +385,32 @@ def test_unicode_non_ascii(self): def test_with_nonstring_type(self): value = object() self.assertRaises(TypeError, self._callFUT, value) + + +class Test__total_seconds_backport(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud_bigtable._non_upstream_helpers import ( + _total_seconds_backport) + return _total_seconds_backport(*args, **kwargs) + + def test_it(self): + import datetime + offset = datetime.timedelta(seconds=3, + microseconds=140000) + result = self._callFUT(offset) + self.assertEqual(result, 3.14) + + +class Test__total_seconds(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud_bigtable._non_upstream_helpers import _total_seconds + return _total_seconds(*args, **kwargs) + + def test_it(self): + import datetime + offset = datetime.timedelta(seconds=1, + microseconds=414000) + result = self._callFUT(offset) + self.assertEqual(result, 1.414) diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py index 9a685ea..c088552 100644 --- a/gcloud_bigtable/test_column_family.py +++ b/gcloud_bigtable/test_column_family.py @@ -23,62 +23,32 @@ COLUMN_FAMILY_ID = 'column-family-id' -class TestGarbageCollectionRule(unittest2.TestCase): +class TestMaxVersionsGCRule(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - return GarbageCollectionRule + from gcloud_bigtable.column_family import MaxVersionsGCRule + return MaxVersionsGCRule def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) - def test_constructor_defaults(self): - gc_rule = self._makeOne() - self.assertEqual(gc_rule.max_num_versions, None) - self.assertEqual(gc_rule.max_age, None) - - def test_constructor_failure(self): - with self.assertRaises(TypeError): - self._makeOne(max_num_versions=1, max_age=object()) - - def test___eq__max_age(self): - max_age = object() - gc_rule1 = self._makeOne(max_age=max_age) - gc_rule2 = self._makeOne(max_age=max_age) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__max_num_versions(self): - gc_rule1 = self._makeOne(max_num_versions=2) - gc_rule2 = self._makeOne(max_num_versions=2) - self.assertEqual(gc_rule1, gc_rule2) - def test___eq__type_differ(self): - gc_rule1 = self._makeOne() + gc_rule1 = self._makeOne(10) gc_rule2 = object() self.assertNotEqual(gc_rule1, gc_rule2) + def test___eq__same_value(self): + gc_rule1 = self._makeOne(2) + gc_rule2 = self._makeOne(2) + self.assertEqual(gc_rule1, gc_rule2) + def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() + gc_rule1 = self._makeOne(99) + gc_rule2 = self._makeOne(99) comparison_val = (gc_rule1 != gc_rule2) self.assertFalse(comparison_val) - def test_to_pb_too_many_values(self): - # Fool the constructor by passing no values. - gc_rule = self._makeOne() - gc_rule.max_num_versions = object() - gc_rule.max_age = object() - with self.assertRaises(TypeError): - gc_rule.to_pb() - - def test_to_pb_no_value(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - gc_rule = self._makeOne() - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, data_pb2.GcRule()) - - def test_to_pb_with_max_num_versions(self): + def test_to_pb(self): from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) max_num_versions = 1337 @@ -87,7 +57,36 @@ def test_to_pb_with_max_num_versions(self): self.assertEqual(pb_val, data_pb2.GcRule(max_num_versions=max_num_versions)) - def test_to_pb_with_max_age(self): + +class TestMaxAgeGCRule(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.column_family import MaxAgeGCRule + return MaxAgeGCRule + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test___eq__type_differ(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = object() + self.assertNotEqual(gc_rule1, gc_rule2) + + def test___eq__same_value(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = self._makeOne(max_age=max_age) + self.assertEqual(gc_rule1, gc_rule2) + + def test___ne__same_value(self): + max_age = object() + gc_rule1 = self._makeOne(max_age=max_age) + gc_rule2 = self._makeOne(max_age=max_age) + comparison_val = (gc_rule1 != gc_rule2) + self.assertFalse(comparison_val) + + def test_to_pb(self): import datetime from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) @@ -100,34 +99,36 @@ def test_to_pb_with_max_age(self): self.assertEqual(pb_val, data_pb2.GcRule(max_age=duration)) -class TestGarbageCollectionRuleUnion(unittest2.TestCase): +class TestGCRuleUnion(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion - return GarbageCollectionRuleUnion + from gcloud_bigtable.column_family import GCRuleUnion + return GCRuleUnion def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def test_constructor(self): rules = object() - rule_union = self._makeOne(rules=rules) + rule_union = self._makeOne(rules) self.assertTrue(rule_union.rules is rules) def test___eq__(self): rules = object() - gc_rule1 = self._makeOne(rules=rules) - gc_rule2 = self._makeOne(rules=rules) + gc_rule1 = self._makeOne(rules) + gc_rule2 = self._makeOne(rules) self.assertEqual(gc_rule1, gc_rule2) def test___eq__type_differ(self): - gc_rule1 = self._makeOne() + rules = object() + gc_rule1 = self._makeOne(rules) gc_rule2 = object() self.assertNotEqual(gc_rule1, gc_rule2) def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() + rules = object() + gc_rule1 = self._makeOne(rules) + gc_rule2 = self._makeOne(rules) comparison_val = (gc_rule1 != gc_rule2) self.assertFalse(comparison_val) @@ -136,14 +137,15 @@ def test_to_pb(self): from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule max_num_versions = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions) + rule1 = MaxVersionsGCRule(max_num_versions) pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) + rule2 = MaxAgeGCRule(max_age) pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) rule3 = self._makeOne(rules=[rule1, rule2]) @@ -158,14 +160,15 @@ def test_to_pb_nested(self): from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule max_num_versions1 = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions1) + rule1 = MaxVersionsGCRule(max_num_versions1) pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) + rule2 = MaxAgeGCRule(max_age) pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) rule3 = self._makeOne(rules=[rule1, rule2]) @@ -173,7 +176,7 @@ def test_to_pb_nested(self): union=data_pb2.GcRule.Union(rules=[pb_rule1, pb_rule2])) max_num_versions2 = 1337 - rule4 = GarbageCollectionRule(max_num_versions=max_num_versions2) + rule4 = MaxVersionsGCRule(max_num_versions2) pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) rule5 = self._makeOne(rules=[rule3, rule4]) @@ -184,35 +187,36 @@ def test_to_pb_nested(self): self.assertEqual(gc_rule_pb, pb_rule5) -class TestGarbageCollectionRuleIntersection(unittest2.TestCase): +class TestGCRuleIntersection(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - return GarbageCollectionRuleIntersection + from gcloud_bigtable.column_family import GCRuleIntersection + return GCRuleIntersection def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def test_constructor(self): rules = object() - rule_intersection = self._makeOne(rules=rules) + rule_intersection = self._makeOne(rules) self.assertTrue(rule_intersection.rules is rules) def test___eq__(self): rules = object() - gc_rule1 = self._makeOne(rules=rules) - gc_rule2 = self._makeOne(rules=rules) + gc_rule1 = self._makeOne(rules) + gc_rule2 = self._makeOne(rules) self.assertEqual(gc_rule1, gc_rule2) def test___eq__type_differ(self): - gc_rule1 = self._makeOne() + rules = object() + gc_rule1 = self._makeOne(rules) gc_rule2 = object() self.assertNotEqual(gc_rule1, gc_rule2) def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() + rules = object() + gc_rule1 = self._makeOne(rules) + gc_rule2 = self._makeOne(rules) comparison_val = (gc_rule1 != gc_rule2) self.assertFalse(comparison_val) @@ -221,14 +225,15 @@ def test_to_pb(self): from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule max_num_versions = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions) + rule1 = MaxVersionsGCRule(max_num_versions) pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) + rule2 = MaxAgeGCRule(max_age) pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) rule3 = self._makeOne(rules=[rule1, rule2]) @@ -244,14 +249,15 @@ def test_to_pb_nested(self): from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule max_num_versions1 = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions1) + rule1 = MaxVersionsGCRule(max_num_versions1) pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) + rule2 = MaxAgeGCRule(max_age) pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) rule3 = self._makeOne(rules=[rule1, rule2]) @@ -260,7 +266,7 @@ def test_to_pb_nested(self): rules=[pb_rule1, pb_rule2])) max_num_versions2 = 1337 - rule4 = GarbageCollectionRule(max_num_versions=max_num_versions2) + rule4 = MaxVersionsGCRule(max_num_versions2) pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) rule5 = self._makeOne(rules=[rule3, rule4]) @@ -289,24 +295,6 @@ def test_constructor(self): self.assertTrue(column_family._table is table) self.assertTrue(column_family.gc_rule is gc_rule) - def test_table_getter(self): - table = object() - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertTrue(column_family.table is table) - - def test_client_getter(self): - client = object() - table = _Table(None, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertTrue(column_family.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 889 - client = _Client(timeout_seconds=timeout_seconds) - table = _Table(None, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertEqual(column_family.timeout_seconds, timeout_seconds) - def test_name_property(self): table_name = 'table_name' table = _Table(table_name) @@ -346,7 +334,8 @@ def _create_test_helper(self, gc_rule=None): bigtable_table_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 4 + client = _Client(timeout_seconds=timeout_seconds) table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) table = _Table(table_name, client=client) @@ -373,8 +362,7 @@ def _create_test_helper(self, gc_rule=None): expected_result = None # create() has no return value. # Perform the method and check the result. - timeout_seconds = 4 - result = column_family.create(timeout_seconds=timeout_seconds) + result = column_family.create() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CreateColumnFamily', @@ -386,8 +374,8 @@ def test_create(self): self._create_test_helper(gc_rule=None) def test_create_with_gc_rule(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - gc_rule = GarbageCollectionRule(max_num_versions=1337) + from gcloud_bigtable.column_family import MaxVersionsGCRule + gc_rule = MaxVersionsGCRule(1337) self._create_test_helper(gc_rule=gc_rule) def _update_test_helper(self, gc_rule=None): @@ -395,7 +383,8 @@ def _update_test_helper(self, gc_rule=None): bigtable_table_data_pb2 as data_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 28 + client = _Client(timeout_seconds=timeout_seconds) table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) table = _Table(table_name, client=client) @@ -421,8 +410,7 @@ def _update_test_helper(self, gc_rule=None): expected_result = None # update() has no return value. # Perform the method and check the result. - timeout_seconds = 28 - result = column_family.update(timeout_seconds=timeout_seconds) + result = column_family.update() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'UpdateColumnFamily', @@ -434,8 +422,8 @@ def test_update(self): self._update_test_helper(gc_rule=None) def test_update_with_gc_rule(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - gc_rule = GarbageCollectionRule(max_num_versions=1337) + from gcloud_bigtable.column_family import MaxVersionsGCRule + gc_rule = MaxVersionsGCRule(1337) self._update_test_helper(gc_rule=gc_rule) def test_delete(self): @@ -444,7 +432,8 @@ def test_delete(self): from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 7 + client = _Client(timeout_seconds=timeout_seconds) table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) table = _Table(table_name, client=client) @@ -465,8 +454,7 @@ def test_delete(self): expected_result = None # delete() has no return value. # Perform the method and check the result. - timeout_seconds = 7 - result = column_family.delete(timeout_seconds=timeout_seconds) + result = column_family.delete() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'DeleteColumnFamily', @@ -477,9 +465,9 @@ def test_delete(self): class Test__gc_rule_from_pb(unittest2.TestCase): - def _callFUT(self, gc_rule_pb): + def _callFUT(self, *args, **kwargs): from gcloud_bigtable.column_family import _gc_rule_from_pb - return _gc_rule_from_pb(gc_rule_pb) + return _gc_rule_from_pb(*args, **kwargs) def test_empty(self): from gcloud_bigtable._generated import ( @@ -488,84 +476,66 @@ def test_empty(self): gc_rule_pb = data_pb2.GcRule() self.assertEqual(self._callFUT(gc_rule_pb), None) - def test_failure(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - - gc_rule_pb1 = data_pb2.GcRule(max_num_versions=1) - gc_rule_pb2 = data_pb2.GcRule( - max_age=duration_pb2.Duration(seconds=1), - ) - # Since a oneof field, google.protobuf doesn't allow both - # to be set, so we fake it. - gc_rule_pb3 = data_pb2.GcRule() - gc_rule_pb3._fields.update(gc_rule_pb1._fields) - gc_rule_pb3._fields.update(gc_rule_pb2._fields) - - with self.assertRaises(ValueError): - self._callFUT(gc_rule_pb3) - def test_max_num_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxVersionsGCRule - orig_rule = GarbageCollectionRule(max_num_versions=1) + orig_rule = MaxVersionsGCRule(1) gc_rule_pb = orig_rule.to_pb() result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRule)) + self.assertTrue(isinstance(result, MaxVersionsGCRule)) self.assertEqual(result, orig_rule) def test_max_age(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule + from gcloud_bigtable.column_family import MaxAgeGCRule - orig_rule = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) + orig_rule = MaxAgeGCRule(datetime.timedelta(seconds=1)) gc_rule_pb = orig_rule.to_pb() result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRule)) + self.assertTrue(isinstance(result, MaxAgeGCRule)) self.assertEqual(result, orig_rule) def test_union(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion + from gcloud_bigtable.column_family import GCRuleUnion + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) - orig_rule = GarbageCollectionRuleUnion(rules=[rule1, rule2]) + rule1 = MaxVersionsGCRule(1) + rule2 = MaxAgeGCRule(datetime.timedelta(seconds=1)) + orig_rule = GCRuleUnion([rule1, rule2]) gc_rule_pb = orig_rule.to_pb() result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRuleUnion)) + self.assertTrue(isinstance(result, GCRuleUnion)) self.assertEqual(result, orig_rule) def test_intersection(self): import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) - orig_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) + from gcloud_bigtable.column_family import GCRuleIntersection + from gcloud_bigtable.column_family import MaxAgeGCRule + from gcloud_bigtable.column_family import MaxVersionsGCRule + + rule1 = MaxVersionsGCRule(1) + rule2 = MaxAgeGCRule(datetime.timedelta(seconds=1)) + orig_rule = GCRuleIntersection([rule1, rule2]) gc_rule_pb = orig_rule.to_pb() result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRuleIntersection)) + self.assertTrue(isinstance(result, GCRuleIntersection)) self.assertEqual(result, orig_rule) def test_unknown_field_name(self): - from google.protobuf.descriptor import FieldDescriptor - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) + class MockProto(object): - gc_rule_pb = data_pb2.GcRule() - fake_descriptor_name = 'not-union' - descriptor_args = (fake_descriptor_name,) + (None,) * 12 - fake_descriptor = FieldDescriptor(*descriptor_args) - gc_rule_pb._fields[fake_descriptor] = None - self.assertEqual(self._callFUT(gc_rule_pb), None) + names = [] + + @classmethod + def WhichOneof(cls, name): + cls.names.append(name) + return 'unknown' + + self.assertEqual(MockProto.names, []) + self.assertRaises(ValueError, self._callFUT, MockProto) + self.assertEqual(MockProto.names, ['rule']) class Test__timedelta_to_duration_pb(unittest2.TestCase): From 0e17f1b2ab18b103f3c107eaa45e3bc6ea8380f3 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 18:45:59 -0800 Subject: [PATCH 35/65] Updating system tests after upstream changes. --- system_tests/run.py | 8 ++++---- system_tests/run_happybase.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/system_tests/run.py b/system_tests/run.py index 8945d91..a0f7985 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -27,7 +27,7 @@ from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import GarbageCollectionRule +from gcloud_bigtable.column_family import MaxVersionsGCRule from gcloud_bigtable.row import RowFilter from gcloud_bigtable.row import RowFilterChain from gcloud_bigtable.row import RowFilterUnion @@ -231,7 +231,7 @@ def test_create_column_family(self): self.tables_to_delete.append(temp_table) self.assertEqual(temp_table.list_column_families(), {}) - gc_rule = GarbageCollectionRule(max_num_versions=1) + gc_rule = MaxVersionsGCRule(1) column_family = temp_table.column_family(COLUMN_FAMILY_ID1, gc_rule=gc_rule) column_family.create() @@ -240,7 +240,7 @@ def test_create_column_family(self): self.assertEqual(len(col_fams), 1) retrieved_col_fam = col_fams[COLUMN_FAMILY_ID1] - self.assertTrue(retrieved_col_fam.table is column_family.table) + self.assertTrue(retrieved_col_fam._table is column_family._table) self.assertEqual(retrieved_col_fam.column_family_id, column_family.column_family_id) self.assertEqual(retrieved_col_fam.gc_rule, gc_rule) @@ -251,7 +251,7 @@ def test_update_column_family(self): temp_table.create() self.tables_to_delete.append(temp_table) - gc_rule = GarbageCollectionRule(max_num_versions=1) + gc_rule = MaxVersionsGCRule(1) column_family = temp_table.column_family(COLUMN_FAMILY_ID1, gc_rule=gc_rule) column_family.create() diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py index 6b6ea86..94296ad 100644 --- a/system_tests/run_happybase.py +++ b/system_tests/run_happybase.py @@ -61,11 +61,12 @@ def set_hbase_connection(): def set_cloud_bigtable_connection(): # pylint: disable=no-name-in-module from grpc.framework.interfaces.face.face import NetworkError + from gcloud_bigtable import _non_upstream_helpers from gcloud_bigtable import client as client_mod from gcloud_bigtable.happybase import Connection # pylint: enable=no-name-in-module - client_mod.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' + _non_upstream_helpers.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' client = client_mod.Client(admin=True) zone = 'us-central1-c' cluster_id = 'gcloud-python' From 9e746da0afc335ea4385a781d2af0c7221473ebe Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 18:49:04 -0800 Subject: [PATCH 36/65] Removing unused logging module. --- gcloud_bigtable/_logging.py | 75 ------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 gcloud_bigtable/_logging.py diff --git a/gcloud_bigtable/_logging.py b/gcloud_bigtable/_logging.py deleted file mode 100644 index 57f0591..0000000 --- a/gcloud_bigtable/_logging.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Logging settings for Google Cloud Bigtable API package.""" - - -import argparse -import logging -import os -import sys - - -LOGGER = logging.getLogger('gcloud_bigtable') -PARSER = argparse.ArgumentParser( - description='Internal gcloud-python-bigtable logging parser.') -# No help since internal. -PARSER.add_argument('--log', dest='log_level') -ENV_VAR_NAME = 'GCLOUD_LOGGING_LEVEL' - -# Get logging level from user input. -DEFAULT_LEVEL = logging.INFO -MAPPING = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARN': logging.WARN, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL, -} - - -def get_log_level(argv): - """Get the logging level from command line arguments. - - :type argv: list of strings - :param argv: Command line arguments from caller. - - :rtype: int - :returns: The logging level parsed from the command line flags. - """ - parsed_args, _ = PARSER.parse_known_args(argv) - # Use upper case since that is used in MAPPING. - log_level = parsed_args.log_level or os.getenv(ENV_VAR_NAME) - return MAPPING.get(log_level, DEFAULT_LEVEL) - - -def setup_logger(logger, argv): - """Set the logging level on a logger. - - Uses ``get_log_level`` from - - :type logger: :class:`logging.Logger` - :param logger: The logger that needs to be set-up. - - :type argv: list of strings - :param argv: Command line arguments from caller. - """ - log_level = get_log_level(argv) - logger.setLevel(log_level) - stream_handler = logging.StreamHandler() - stream_handler.setLevel(log_level) - logger.addHandler(stream_handler) - - -setup_logger(LOGGER, sys.argv) From 0c58512ccafb6aaea8af09c69fa85e955e85084e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 18:51:38 -0800 Subject: [PATCH 37/65] Bringing local __init__.py closer to upstream. This is so that a simple diff is possible. --- gcloud_bigtable/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/gcloud_bigtable/__init__.py b/gcloud_bigtable/__init__.py index 8393025..f26e598 100644 --- a/gcloud_bigtable/__init__.py +++ b/gcloud_bigtable/__init__.py @@ -14,15 +14,18 @@ """Google Cloud Bigtable API package.""" + +from __future__ import print_function +import sys + try: from grpc._adapter import _c -except ImportError as exc: +except ImportError as exc: # pragma: NO COVER if 'libgrpc.so' in str(exc): - raise ImportError('gRPC libraries could not be located. Please see ' - 'instructions to locate these files. You\'ll want ' - 'to set your LD_LIBRARY_PATH variable to help ' - 'Python locate the libraries.') - else: - raise + print('gRPC libraries could not be located. Please see ' + 'instructions to locate these files. You\'ll want ' + 'to set your LD_LIBRARY_PATH variable to help ' + 'Python locate the libraries.', file=sys.stderr) + raise from gcloud_bigtable.client import Client From 764df9f036cdb6ee164caa58c8d3e84215a4864f Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 18:57:59 -0800 Subject: [PATCH 38/65] Moving _parse_family_pb from _helpers into row. This is because that was the only place it was used. Also remove test___init__ since we don't fail in the same way any longer. --- gcloud_bigtable/_helpers.py | 40 --------- gcloud_bigtable/row.py | 40 ++++++++- gcloud_bigtable/test___init__.py | 136 ------------------------------- gcloud_bigtable/test__helpers.py | 60 -------------- gcloud_bigtable/test_row.py | 60 ++++++++++++++ 5 files changed, 99 insertions(+), 237 deletions(-) delete mode 100644 gcloud_bigtable/test___init__.py diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index c19371e..45c1a30 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -23,8 +23,6 @@ from grpc.beta import implementations -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp - # See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 # for appropriate values on other systems. @@ -106,41 +104,3 @@ def make_stub(client, stub_factory, host, port): custom_metadata_transformer = MetadataTransformer(client) return stub_factory(channel, metadata_transformer=custom_metadata_transformer) - - -def _parse_family_pb(family_pb): - """Parses a Family protobuf into a dictionary. - - :type family_pb: :class:`._generated.bigtable_data_pb2.Family` - :param family_pb: A protobuf - - :rtype: tuple - :returns: A string and dictionary. The string is the name of the - column family and the dictionary has column names (within the - family) as keys and cell lists as values. Each cell is - represented with a two-tuple with the value (in bytes) and the - timestamp for the cell. For example: - - .. code:: python - - { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - } - """ - result = {} - for column in family_pb.columns: - result[column.qualifier] = cells = [] - for cell in column.cells: - val_pair = ( - cell.value, - _microseconds_to_timestamp(cell.timestamp_micros), - ) - cells.append(val_pair) - - return family_pb.name, result diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 044082d..fc17986 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -22,7 +22,7 @@ from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _parse_family_pb +from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable._non_upstream_helpers import _to_bytes @@ -1205,3 +1205,41 @@ def _parse_rmw_row_response(row_response): column_family_id, curr_family = _parse_family_pb(column_family) result[column_family_id] = curr_family return result + + +def _parse_family_pb(family_pb): + """Parses a Family protobuf into a dictionary. + + :type family_pb: :class:`._generated.bigtable_data_pb2.Family` + :param family_pb: A protobuf + + :rtype: tuple + :returns: A string and dictionary. The string is the name of the + column family and the dictionary has column names (within the + family) as keys and cell lists as values. Each cell is + represented with a two-tuple with the value (in bytes) and the + timestamp for the cell. For example: + + .. code:: python + + { + b'col-name1': [ + (b'cell-val', datetime.datetime(...)), + (b'cell-val-newer', datetime.datetime(...)), + ], + b'col-name2': [ + (b'altcol-cell-val', datetime.datetime(...)), + ], + } + """ + result = {} + for column in family_pb.columns: + result[column.qualifier] = cells = [] + for cell in column.cells: + val_pair = ( + cell.value, + _microseconds_to_timestamp(cell.timestamp_micros), + ) + cells.append(val_pair) + + return family_pb.name, result diff --git a/gcloud_bigtable/test___init__.py b/gcloud_bigtable/test___init__.py deleted file mode 100644 index e9d9afc..0000000 --- a/gcloud_bigtable/test___init__.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestGRPCImportFailure(unittest2.TestCase): - - _MODULES_TO_EDIT = ('gcloud_bigtable', 'grpc', - 'grpc._adapter', 'grpc._adapter._c') - - @classmethod - def setUpClass(cls): - import imp - mod_name = 'gcloud_bigtable' - # H/T: http://pymotw.com/2/imp/ - cls._load_module_args = (mod_name,) + imp.find_module(mod_name) - - @classmethod - def tearDownClass(cls): - del cls._load_module_args - - def _module_patch_helper(self, function_to_test): - import sys - - removed_mods = {} - for mod_name in self._MODULES_TO_EDIT: - removed_mods[mod_name] = sys.modules.get(mod_name) - - try: - sys.modules.pop('gcloud_bigtable', None) - # Test method should re-import gcloud_bigtable - function_to_test() - finally: - for mod_name, value in removed_mods.items(): - sys.modules[mod_name] = value - - def test_success(self): - import imp - import sys - import types - - TEST_CASE = self - - def function_to_test(): - c_mod = types.ModuleType('grpc._adapter._c') - sys.modules['grpc._adapter._c'] = c_mod - - adapter_mod = types.ModuleType('grpc._adapter') - adapter_mod._c = c_mod - sys.modules['grpc._adapter'] = adapter_mod - - grpc_mod = types.ModuleType('grpc') - grpc_mod._adapter = adapter_mod - sys.modules['grpc'] = grpc_mod - - # Re-import gcloud_bigtable and check our custom _c module. - gcloud_bigtable = imp.load_module(*TEST_CASE._load_module_args) - TEST_CASE.assertTrue(gcloud_bigtable._c is c_mod) - - self._module_patch_helper(function_to_test) - - @staticmethod - def _create_fake_grpc_adapter_package(contents): - import os - import tempfile - - temp_dir = tempfile.mkdtemp() - - curr_dir = os.path.join(temp_dir, 'grpc') - os.mkdir(curr_dir) - with open(os.path.join(curr_dir, '__init__.py'), 'wb') as file_obj: - file_obj.write(b'') - - curr_dir = os.path.join(curr_dir, '_adapter') - os.mkdir(curr_dir) - with open(os.path.join(curr_dir, '__init__.py'), 'wb') as file_obj: - file_obj.write(b'') - - filename = os.path.join(curr_dir, '_c.py') - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - return temp_dir - - def _import_fail_helper(self, orig_exc_message): - import imp - import sys - - # Be sure the message contains libgrpc.so - c_module_contents = 'raise ImportError(%r)\n' % (orig_exc_message,) - c_module_contents = c_module_contents.encode('ascii') - temp_dir = self._create_fake_grpc_adapter_package(c_module_contents) - - TEST_CASE = self - - def function_to_test(): - sys.path.insert(0, temp_dir) - try: - sys.modules.pop('grpc') - sys.modules.pop('grpc._adapter') - sys.modules.pop('grpc._adapter._c') - - try: - imp.load_module(*TEST_CASE._load_module_args) - except ImportError as exc: - if 'libgrpc.so' in orig_exc_message: - TEST_CASE.assertNotEqual(str(exc), orig_exc_message) - else: - TEST_CASE.assertEqual(str(exc), orig_exc_message) - finally: - sys.path.remove(temp_dir) - - self._module_patch_helper(function_to_test) - - def test_system_library_cause(self): - # Be sure the message contains libgrpc.so - orig_exc_message = 'bad libgrpc.so' - self._import_fail_helper(orig_exc_message) - - def test_non_system_library_cause(self): - # Be sure the message does not contain libgrpc.so - orig_exc_message = 'Other cause of error' - self._import_fail_helper(orig_exc_message) diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index c9ef90a..7d3b1c5 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -142,63 +142,3 @@ def test_it(self): {'private_key': None, 'certificate_chain': None}), ('secure_channel', (host, port, client_creds), {}), ]) - - -class Test__parse_family_pb(unittest2.TestCase): - - def _callFUT(self, family_pb): - from gcloud_bigtable._helpers import _parse_family_pb - return _parse_family_pb(family_pb) - - def test_it(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - - COL_FAM1 = u'col-fam-id' - COL_NAME1 = b'col-name1' - COL_NAME2 = b'col-name2' - CELL_VAL1 = b'cell-val' - CELL_VAL2 = b'cell-val-newer' - CELL_VAL3 = b'altcol-cell-val' - - microseconds = 5554441037 - timestamp = _microseconds_to_timestamp(microseconds) - expected_dict = { - COL_NAME1: [ - (CELL_VAL1, timestamp), - (CELL_VAL2, timestamp), - ], - COL_NAME2: [ - (CELL_VAL3, timestamp), - ], - } - expected_output = (COL_FAM1, expected_dict) - sample_input = data_pb2.Family( - name=COL_FAM1, - columns=[ - data_pb2.Column( - qualifier=COL_NAME1, - cells=[ - data_pb2.Cell( - value=CELL_VAL1, - timestamp_micros=microseconds, - ), - data_pb2.Cell( - value=CELL_VAL2, - timestamp_micros=microseconds, - ), - ], - ), - data_pb2.Column( - qualifier=COL_NAME2, - cells=[ - data_pb2.Cell( - value=CELL_VAL3, - timestamp_micros=microseconds, - ), - ], - ), - ], - ) - self.assertEqual(expected_output, self._callFUT(sample_input)) diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index d3cbd75..d797ecc 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -1524,6 +1524,66 @@ def test_to_pb_false_only(self): self.assertEqual(filter_pb, expected_pb) +class Test__parse_family_pb(unittest2.TestCase): + + def _callFUT(self, family_pb): + from gcloud_bigtable.row import _parse_family_pb + return _parse_family_pb(family_pb) + + def test_it(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud_bigtable._non_upstream_helpers import ( + _microseconds_to_timestamp) + + COL_FAM1 = u'col-fam-id' + COL_NAME1 = b'col-name1' + COL_NAME2 = b'col-name2' + CELL_VAL1 = b'cell-val' + CELL_VAL2 = b'cell-val-newer' + CELL_VAL3 = b'altcol-cell-val' + + microseconds = 5554441037 + timestamp = _microseconds_to_timestamp(microseconds) + expected_dict = { + COL_NAME1: [ + (CELL_VAL1, timestamp), + (CELL_VAL2, timestamp), + ], + COL_NAME2: [ + (CELL_VAL3, timestamp), + ], + } + expected_output = (COL_FAM1, expected_dict) + sample_input = data_pb2.Family( + name=COL_FAM1, + columns=[ + data_pb2.Column( + qualifier=COL_NAME1, + cells=[ + data_pb2.Cell( + value=CELL_VAL1, + timestamp_micros=microseconds, + ), + data_pb2.Cell( + value=CELL_VAL2, + timestamp_micros=microseconds, + ), + ], + ), + data_pb2.Column( + qualifier=COL_NAME2, + cells=[ + data_pb2.Cell( + value=CELL_VAL3, + timestamp_micros=microseconds, + ), + ], + ), + ], + ) + self.assertEqual(expected_output, self._callFUT(sample_input)) + + class _Client(object): data_stub = None From d6164dc5bdb81ba1770393dcf666a53cc405458e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 20:27:13 -0800 Subject: [PATCH 39/65] Pulling in individualized RowFilters from upstream. --- gcloud_bigtable/happybase/table.py | 31 +- gcloud_bigtable/happybase/test_table.py | 69 +- gcloud_bigtable/row.py | 789 +++++++++++------- gcloud_bigtable/test_row.py | 1018 +++++++++++++---------- gcloud_bigtable/test_table.py | 4 +- pylintrc_default | 2 +- 6 files changed, 1113 insertions(+), 800 deletions(-) diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index 4f18470..66f51da 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -28,10 +28,14 @@ from gcloud_bigtable.happybase.batch import Batch from gcloud_bigtable.happybase.batch import _WAL_SENTINEL from gcloud_bigtable.happybase.batch import _get_column_pairs -from gcloud_bigtable.row import RowFilter +from gcloud_bigtable.row import CellsColumnLimitFilter +from gcloud_bigtable.row import ColumnQualifierRegexFilter +from gcloud_bigtable.row import FamilyNameRegexFilter from gcloud_bigtable.row import RowFilterChain from gcloud_bigtable.row import RowFilterUnion +from gcloud_bigtable.row import RowKeyRegexFilter from gcloud_bigtable.row import TimestampRange +from gcloud_bigtable.row import TimestampRangeFilter from gcloud_bigtable.table import Table as _LowLevelTable @@ -204,7 +208,7 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, :type filters: list :param filters: (Optional) List of existing filters to be extended. - :rtype: :class:`.RowFilterChain`, :class:`.RowFilter` + :rtype: :class:`.RowFilter` :returns: The chained filter created, or just a single filter if only one was needed. :raises: :class:`ValueError ` if there are no @@ -215,14 +219,14 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, if column is not None: column_family_id, column_qualifier = column.split(':') - fam_filter = RowFilter(family_name_regex_filter=column_family_id) - qual_filter = RowFilter(column_qualifier_regex_filter=column_qualifier) + fam_filter = FamilyNameRegexFilter(column_family_id) + qual_filter = ColumnQualifierRegexFilter(column_qualifier) filters.extend([fam_filter, qual_filter]) if versions is not None: - filters.append(RowFilter(cells_per_column_limit_filter=versions)) + filters.append(CellsColumnLimitFilter(versions)) time_range = _convert_to_time_range(timestamp=timestamp) if time_range is not None: - filters.append(RowFilter(timestamp_range_filter=time_range)) + filters.append(TimestampRangeFilter(time_range)) num_filters = len(filters) if num_filters == 0: @@ -243,22 +247,20 @@ def _columns_filter_helper(columns): * an entire column family: ``fam`` or ``fam:`` * an single column: ``fam:col`` - :rtype: :class:`.RowFilterUnion`, :class:`.RowFilter` + :rtype: :class:`.RowFilter` :returns: The union filter created containing all of the matched columns. :raises: :class:`ValueError ` if there are no filters to union. """ filters = [] for column_family_id, column_qualifier in _get_column_pairs(columns): + fam_filter = FamilyNameRegexFilter(column_family_id) if column_qualifier is not None: - fam_filter = RowFilter(family_name_regex_filter=column_family_id) - qual_filter = RowFilter( - column_qualifier_regex_filter=column_qualifier) + qual_filter = ColumnQualifierRegexFilter(column_qualifier) combined_filter = RowFilterChain( filters=[fam_filter, qual_filter]) filters.append(combined_filter) else: - fam_filter = RowFilter(family_name_regex_filter=column_family_id) filters.append(fam_filter) num_filters = len(filters) @@ -276,14 +278,14 @@ def _row_keys_filter_helper(row_keys): :type row_keys: list :param row_keys: Iterable containing row keys (as strings). - :rtype: :class:`.RowFilterUnion`, :class:`.RowFilter` + :rtype: :class:`.RowFilter` :returns: The union filter created containing all of the row keys. :raises: :class:`ValueError ` if there are no filters to union. """ filters = [] for row_key in row_keys: - filters.append(RowFilter(row_key_regex_filter=row_key)) + filters.append(RowKeyRegexFilter(row_key)) num_filters = len(filters) if num_filters == 0: @@ -618,8 +620,7 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, * an entire column family: ``fam`` or ``fam:`` * an single column: ``fam:col`` - :type filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` + :type filter: :class:`.RowFilter` :param filter: (Optional) An additional filter (beyond column and row range filters supported here). HappyBase / HBase users will have used this as an HBase filter string. See diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py index 89a4921..65774c1 100644 --- a/gcloud_bigtable/happybase/test_table.py +++ b/gcloud_bigtable/happybase/test_table.py @@ -204,17 +204,17 @@ def test_no_filters(self): self._callFUT() def test_single_filter(self): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsColumnLimitFilter versions = 1337 result = self._callFUT(versions=versions) - self.assertTrue(isinstance(result, RowFilter)) + self.assertTrue(isinstance(result, CellsColumnLimitFilter)) # Relies on the fact that RowFilter instances can # only have one value set. - self.assertEqual(result.cells_per_column_limit_filter, versions) + self.assertEqual(result.num_cells, versions) def test_existing_filters(self): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsColumnLimitFilter filters = [] versions = 1337 @@ -222,13 +222,14 @@ def test_existing_filters(self): # Make sure filters has grown. self.assertEqual(filters, [result]) - self.assertTrue(isinstance(result, RowFilter)) + self.assertTrue(isinstance(result, CellsColumnLimitFilter)) # Relies on the fact that RowFilter instances can # only have one value set. - self.assertEqual(result.cells_per_column_limit_filter, versions) + self.assertEqual(result.num_cells, versions) def _column_helper(self, num_filters, versions=None, timestamp=None): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import ColumnQualifierRegexFilter + from gcloud_bigtable.row import FamilyNameRegexFilter from gcloud_bigtable.row import RowFilterChain col_fam = 'cf1' @@ -240,13 +241,13 @@ def _column_helper(self, num_filters, versions=None, timestamp=None): self.assertEqual(len(result.filters), num_filters) fam_filter = result.filters[0] qual_filter = result.filters[1] - self.assertTrue(isinstance(fam_filter, RowFilter)) - self.assertTrue(isinstance(qual_filter, RowFilter)) + self.assertTrue(isinstance(fam_filter, FamilyNameRegexFilter)) + self.assertTrue(isinstance(qual_filter, ColumnQualifierRegexFilter)) # Relies on the fact that RowFilter instances can # only have one value set. - self.assertEqual(fam_filter.family_name_regex_filter, col_fam) - self.assertEqual(qual_filter.column_qualifier_regex_filter, qual) + self.assertEqual(fam_filter.regex, col_fam) + self.assertEqual(qual_filter.regex, qual) return result @@ -254,32 +255,31 @@ def test_column_only(self): self._column_helper(num_filters=2) def test_with_versions(self): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsColumnLimitFilter versions = 11 result = self._column_helper(num_filters=3, versions=versions) version_filter = result.filters[2] - self.assertTrue(isinstance(version_filter, RowFilter)) + self.assertTrue(isinstance(version_filter, CellsColumnLimitFilter)) # Relies on the fact that RowFilter instances can # only have one value set. - self.assertEqual(version_filter.cells_per_column_limit_filter, - versions) + self.assertEqual(version_filter.num_cells, versions) def test_with_timestamp(self): from gcloud_bigtable._non_upstream_helpers import ( _microseconds_to_timestamp) - from gcloud_bigtable.row import RowFilter from gcloud_bigtable.row import TimestampRange + from gcloud_bigtable.row import TimestampRangeFilter timestamp = 1441928298571 result = self._column_helper(num_filters=3, timestamp=timestamp) range_filter = result.filters[2] - self.assertTrue(isinstance(range_filter, RowFilter)) + self.assertTrue(isinstance(range_filter, TimestampRangeFilter)) # Relies on the fact that RowFilter instances can # only have one value set. - time_range = range_filter.timestamp_range_filter + time_range = range_filter.range_ self.assertTrue(isinstance(time_range, TimestampRange)) self.assertEqual(time_range.start, None) ts_dt = _microseconds_to_timestamp(1000 * timestamp) @@ -304,16 +304,17 @@ def test_no_columns(self): self._callFUT(columns) def test_single_column(self): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import FamilyNameRegexFilter col_fam = 'cf1' columns = [col_fam] result = self._callFUT(columns) - expected_result = RowFilter(family_name_regex_filter=col_fam) + expected_result = FamilyNameRegexFilter(col_fam) self.assertEqual(result, expected_result) - def test_column_and_column_familieis(self): - from gcloud_bigtable.row import RowFilter + def test_column_and_column_families(self): + from gcloud_bigtable.row import ColumnQualifierRegexFilter + from gcloud_bigtable.row import FamilyNameRegexFilter from gcloud_bigtable.row import RowFilterChain from gcloud_bigtable.row import RowFilterUnion @@ -328,15 +329,15 @@ def test_column_and_column_familieis(self): filter1 = result.filters[0] filter2 = result.filters[1] - self.assertTrue(isinstance(filter1, RowFilter)) - self.assertEqual(filter1.family_name_regex_filter, col_fam1) + self.assertTrue(isinstance(filter1, FamilyNameRegexFilter)) + self.assertEqual(filter1.regex, col_fam1) self.assertTrue(isinstance(filter2, RowFilterChain)) filter2a, filter2b = filter2.filters - self.assertTrue(isinstance(filter2a, RowFilter)) - self.assertEqual(filter2a.family_name_regex_filter, col_fam2) - self.assertTrue(isinstance(filter2b, RowFilter)) - self.assertEqual(filter2b.column_qualifier_regex_filter, col_qual2) + self.assertTrue(isinstance(filter2a, FamilyNameRegexFilter)) + self.assertEqual(filter2a.regex, col_fam2) + self.assertTrue(isinstance(filter2b, ColumnQualifierRegexFilter)) + self.assertEqual(filter2b.regex, col_qual2) class Test__row_keys_filter_helper(unittest2.TestCase): @@ -351,17 +352,17 @@ def test_no_rows(self): self._callFUT(row_keys) def test_single_row(self): - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowKeyRegexFilter row_key = b'row-key' row_keys = [row_key] result = self._callFUT(row_keys) - expected_result = RowFilter(row_key_regex_filter=row_key) + expected_result = RowKeyRegexFilter(row_key) self.assertEqual(result, expected_result) def test_many_rows(self): - from gcloud_bigtable.row import RowFilter from gcloud_bigtable.row import RowFilterUnion + from gcloud_bigtable.row import RowKeyRegexFilter row_key1 = b'row-key1' row_key2 = b'row-key2' @@ -369,9 +370,9 @@ def test_many_rows(self): row_keys = [row_key1, row_key2, row_key3] result = self._callFUT(row_keys) - filter1 = RowFilter(row_key_regex_filter=row_key1) - filter2 = RowFilter(row_key_regex_filter=row_key2) - filter3 = RowFilter(row_key_regex_filter=row_key3) + filter1 = RowKeyRegexFilter(row_key1) + filter2 = RowKeyRegexFilter(row_key2) + filter3 = RowKeyRegexFilter(row_key3) expected_result = RowFilterUnion(filters=[filter1, filter2, filter3]) self.assertEqual(result, expected_result) diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index fc17986..2d00b50 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -27,6 +27,10 @@ from gcloud_bigtable._non_upstream_helpers import _to_bytes +# pylint: disable=invalid-name +# Alias for easy upstream diffs. +_microseconds_from_datetime = _timestamp_to_microseconds +# pylint: enable=invalid-name _MAX_MUTATIONS = 100000 _PACK_I64 = struct.Struct('>q').pack @@ -50,8 +54,7 @@ class Row(object): :type table: :class:`Table ` :param table: The table that owns the row. - :type filter_: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` + :type filter_: :class:`RowFilter` :param filter_: (Optional) Filter to be used for conditional mutations. If a filter is set, then the :class:`Row` will accumulate mutations for either a :data:`True` or :data:`False` state. @@ -99,9 +102,7 @@ def row_key(self): def filter(self): """Getter for row's filter. - :rtype: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion`, :class:`ConditionalRowFilter` or - :data:`NoneType ` + :rtype: :class:`RowFilter` or :data:`NoneType ` :returns: The filter for the row. """ return self._filter @@ -575,279 +576,222 @@ def commit_modifications(self, timeout_seconds=None): return _parse_rmw_row_response(row_response) -# NOTE: For developers, this class may seem to be a bit verbose, i.e. -# a list of property names and **kwargs may do the trick better -# than actually listing every single argument. However, for the sake -# of users and documentation, listing every single argument is more -# useful. -# pylint: disable=too-many-instance-attributes -# pylint: disable=too-many-arguments -# pylint: disable=too-many-branches class RowFilter(object): """Basic filter to apply to cells in a row. These values can be combined via :class:`RowFilterChain`, :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - The regex filters must be valid RE2 patterns. See Google's + .. note:: + + This class is a do-nothing base class for all row filters. + """ + + def __ne__(self, other): + return not self.__eq__(other) + + +class _BoolFilter(RowFilter): + """Row filter that uses a boolean flag. + + :type flag: bool + :param flag: An indicator if a setting is turned on or off. + """ + + def __init__(self, flag): + self.flag = flag + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.flag == self.flag + + +class SinkFilter(_BoolFilter): + """Advanced row filter to skip parent filters. + + :type flag: bool + :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. + Outputs all cells directly to the output of the read rather + than to any parent filter. Cannot be used within the + ``predicate_filter``, ``true_filter``, or ``false_filter`` + of a :class:`ConditionalRowFilter`. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(sink=self.flag) + + +class PassAllFilter(_BoolFilter): + """Row filter equivalent to not filtering at all. + + :type flag: bool + :param flag: Matches all cells, regardless of input. Functionally + equivalent to leaving ``filter`` unset, but included for + completeness. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(pass_all_filter=self.flag) + + +class BlockAllFilter(_BoolFilter): + """Row filter that doesn't match any cells. + + :type flag: bool + :param flag: Does not match any cells, regardless of input. Useful for + temporarily disabling just part of a filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(block_all_filter=self.flag) + + +class _RegexFilter(RowFilter): + """Row filter that uses a regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's `RE2 reference`_ for the accepted syntax. .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - .. note:: + :type regex: bytes or str + :param regex: A regular expression (RE2) for some row filter. + """ + + def __init__(self, regex): + self.regex = regex + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.regex == self.regex - At most one of the keyword arguments can be specified at once. + +class RowKeyRegexFilter(_RegexFilter): + """Row filter for a row key regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax .. note:: - For :class:`bytes` regex filters (``row_key``, ``column_qualifier`` and - ``value``), special care need be used with the expression used. Since + Special care need be used with the expression used. Since each of these properties can contain arbitrary bytes, the ``\\C`` escape sequence must be used if a true wildcard is desired. The ``.`` character will not match the new line character ``\\n``, which may be present in a binary value. - :type sink: bool - :param sink: ADVANCED USE ONLY. Hook for introspection into the RowFilter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - - :type pass_all_filter: bool - :param pass_all_filter: Matches all cells, regardless of input. - Functionally equivalent to leaving ``filter`` - unset, but included for completeness. - - :type block_all_filter: bool - :param block_all_filter: Does not match any cells, regardless of input. - Useful for temporarily disabling just part of - a filter. - - :type row_key_regex_filter: bytes - :param row_key_regex_filter: A regular expression (RE2) to match cells from - rows with row keys that satisfy this regex. - For a ``CheckAndMutateRowRequest``, this - filter is unnecessary since the row key is - already specified. - - :type row_sample_filter: float - :param row_sample_filter: Non-deterministic filter. Matches all cells from - a row with probability p, and matches no cells - from the row with probability 1-p. (Here, the - probability p is ``row_sample_filter``.) - - :type family_name_regex_filter: str - :param family_name_regex_filter: A regular expression (RE2) to match cells - from columns in a given column family. For - technical reasons, the regex must not - contain the ``':'`` character, even if it - isnot being uses as a literal. - - :type column_qualifier_regex_filter: bytes - :param column_qualifier_regex_filter: A regular expression (RE2) to match - cells from column that match this - regex (irrespective of column - family). - - :type column_range_filter: :class:`ColumnRange` - :param column_range_filter: Range of columns to limit cells to. - - :type timestamp_range_filter: :class:`TimestampRange` - :param timestamp_range_filter: Range of time that cells should match - against. - - :type value_regex_filter: bytes - :param value_regex_filter: A regular expression (RE2) to match cells with - values that match this regex. - - :type value_range_filter: :class:`CellValueRange` - :param value_range_filter: Range of cell values to filter for. - - :type cells_per_row_offset_filter: int - :param cells_per_row_offset_filter: Skips the first N cells of the row. - - :type cells_per_row_limit_filter: int - :param cells_per_row_limit_filter: Matches only the first N cells of the - row. - - :type cells_per_column_limit_filter: int - :param cells_per_column_limit_filter: Matches only the most recent N cells - within each column. This filters a - (family name, column) pair, based on - timestamps of each cell. - - :type strip_value_transformer: bool - :param strip_value_transformer: If :data:`True`, replaces each cell's value - with the empty string. As the name - indicates, this is more useful as a - transformer than a generic query / filter. - - :type apply_label_transformer: str - :param apply_label_transformer: Applies the given label to all cells in the - output row. This allows the client to - determine which results were produced from - which part of the filter. - - Values must be at most 15 characters long, - and match the pattern [a-z0-9\\-]+. - - Due to a technical limitation, it is not - currently possible to apply multiple labels - to a cell. - - :raises: :class:`TypeError ` if not exactly one - value set in the constructor. + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from rows with row + keys that satisfy this regex. For a + ``CheckAndMutateRowRequest``, this filter is unnecessary + since the row key is already specified. """ - def __init__(self, - sink=None, - pass_all_filter=None, - block_all_filter=None, - row_key_regex_filter=None, - row_sample_filter=None, - family_name_regex_filter=None, - column_qualifier_regex_filter=None, - column_range_filter=None, - timestamp_range_filter=None, - value_regex_filter=None, - value_range_filter=None, - cells_per_row_offset_filter=None, - cells_per_row_limit_filter=None, - cells_per_column_limit_filter=None, - strip_value_transformer=None, - apply_label_transformer=None): - self.sink = sink - self.pass_all_filter = pass_all_filter - self.block_all_filter = block_all_filter - self.row_key_regex_filter = row_key_regex_filter - self.row_sample_filter = row_sample_filter - self.family_name_regex_filter = family_name_regex_filter - self.column_qualifier_regex_filter = column_qualifier_regex_filter - self.column_range_filter = column_range_filter - self.timestamp_range_filter = timestamp_range_filter - self.value_regex_filter = value_regex_filter - self.value_range_filter = value_range_filter - self.cells_per_row_offset_filter = cells_per_row_offset_filter - self.cells_per_row_limit_filter = cells_per_row_limit_filter - self.cells_per_column_limit_filter = cells_per_column_limit_filter - self.strip_value_transformer = strip_value_transformer - self.apply_label_transformer = apply_label_transformer - self._check_single_value() - - def _check_single_value(self): - """Checks that exactly one value is set on the instance. - - :raises: :class:`TypeError ` if not exactly one - value set on the instance. + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. """ - values_set = ( - int(self.sink is not None) + - int(self.pass_all_filter is not None) + - int(self.block_all_filter is not None) + - int(self.row_key_regex_filter is not None) + - int(self.row_sample_filter is not None) + - int(self.family_name_regex_filter is not None) + - int(self.column_qualifier_regex_filter is not None) + - int(self.column_range_filter is not None) + - int(self.timestamp_range_filter is not None) + - int(self.value_regex_filter is not None) + - int(self.value_range_filter is not None) + - int(self.cells_per_row_offset_filter is not None) + - int(self.cells_per_row_limit_filter is not None) + - int(self.cells_per_column_limit_filter is not None) + - int(self.strip_value_transformer is not None) + - int(self.apply_label_transformer is not None) - ) - if values_set != 1: - raise TypeError('Exactly one value must be set in a row filter') + return data_pb2.RowFilter(row_key_regex_filter=self.regex) + + +class RowSampleFilter(RowFilter): + """Matches all cells from a row with probability p. + + :type sample: float + :param sample: The probability of matching a cell (must be in the + interval ``[0, 1]``). + """ + + def __init__(self, sample): + self.sample = sample def __eq__(self, other): if not isinstance(other, self.__class__): return False - return ( - other.sink == self.sink and - other.pass_all_filter == self.pass_all_filter and - other.block_all_filter == self.block_all_filter and - other.row_key_regex_filter == self.row_key_regex_filter and - other.row_sample_filter == self.row_sample_filter and - other.family_name_regex_filter == self.family_name_regex_filter and - (other.column_qualifier_regex_filter == - self.column_qualifier_regex_filter) and - other.column_range_filter == self.column_range_filter and - other.timestamp_range_filter == self.timestamp_range_filter and - other.value_regex_filter == self.value_regex_filter and - other.value_range_filter == self.value_range_filter and - (other.cells_per_row_offset_filter == - self.cells_per_row_offset_filter) and - (other.cells_per_row_limit_filter == - self.cells_per_row_limit_filter) and - (other.cells_per_column_limit_filter == - self.cells_per_column_limit_filter) and - other.strip_value_transformer == self.strip_value_transformer and - other.apply_label_transformer == self.apply_label_transformer - ) + return other.sample == self.sample - def __ne__(self, other): - return not self.__eq__(other) + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(row_sample_filter=self.sample) + + +class FamilyNameRegexFilter(_RegexFilter): + """Row filter for a family name regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + :type regex: str + :param regex: A regular expression (RE2) to match cells from columns in a + given column family. For technical reasons, the regex must + not contain the ``':'`` character, even if it is not being + used as a literal. + """ def to_pb(self): - """Converts the :class:`RowFilter` to a protobuf. + """Converts the row filter to a protobuf. :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. """ - self._check_single_value() - row_filter_kwargs = {} - if self.sink is not None: - row_filter_kwargs['sink'] = self.sink - if self.pass_all_filter is not None: - row_filter_kwargs['pass_all_filter'] = self.pass_all_filter - if self.block_all_filter is not None: - row_filter_kwargs['block_all_filter'] = self.block_all_filter - if self.row_key_regex_filter is not None: - row_filter_kwargs['row_key_regex_filter'] = _to_bytes( - self.row_key_regex_filter) - if self.row_sample_filter is not None: - row_filter_kwargs['row_sample_filter'] = ( - self.row_sample_filter) - if self.family_name_regex_filter is not None: - row_filter_kwargs['family_name_regex_filter'] = ( - self.family_name_regex_filter) - if self.column_qualifier_regex_filter is not None: - row_filter_kwargs['column_qualifier_regex_filter'] = _to_bytes( - self.column_qualifier_regex_filter) - if self.column_range_filter is not None: - row_filter_kwargs['column_range_filter'] = ( - self.column_range_filter.to_pb()) - if self.timestamp_range_filter is not None: - row_filter_kwargs['timestamp_range_filter'] = ( - self.timestamp_range_filter.to_pb()) - if self.value_regex_filter is not None: - row_filter_kwargs['value_regex_filter'] = _to_bytes( - self.value_regex_filter) - if self.value_range_filter is not None: - row_filter_kwargs['value_range_filter'] = ( - self.value_range_filter.to_pb()) - if self.cells_per_row_offset_filter is not None: - row_filter_kwargs['cells_per_row_offset_filter'] = ( - self.cells_per_row_offset_filter) - if self.cells_per_row_limit_filter is not None: - row_filter_kwargs['cells_per_row_limit_filter'] = ( - self.cells_per_row_limit_filter) - if self.cells_per_column_limit_filter is not None: - row_filter_kwargs['cells_per_column_limit_filter'] = ( - self.cells_per_column_limit_filter) - if self.strip_value_transformer is not None: - row_filter_kwargs['strip_value_transformer'] = ( - self.strip_value_transformer) - if self.apply_label_transformer is not None: - row_filter_kwargs['apply_label_transformer'] = ( - self.apply_label_transformer) - return data_pb2.RowFilter(**row_filter_kwargs) -# pylint: enable=too-many-instance-attributes -# pylint: enable=too-many-arguments -# pylint: enable=too-many-branches + return data_pb2.RowFilter(family_name_regex_filter=self.regex) + + +class ColumnQualifierRegexFilter(_RegexFilter): + """Row filter for a column qualifier regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. + + :type regex: bytes + :param regex: A regular expression (RE2) to match cells from column that + match this regex (irrespective of column family). + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) class TimestampRange(object): @@ -859,7 +803,7 @@ class TimestampRange(object): :type end: :class:`datetime.datetime` :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, defaults to "infinity" (no upper bound). + range. If omitted, no upper bound is used. """ def __init__(self, start=None, end=None): @@ -884,15 +828,42 @@ def to_pb(self): timestamp_range_kwargs = {} if self.start is not None: timestamp_range_kwargs['start_timestamp_micros'] = ( - _timestamp_to_microseconds(self.start)) + _microseconds_from_datetime(self.start)) if self.end is not None: timestamp_range_kwargs['end_timestamp_micros'] = ( - _timestamp_to_microseconds(self.end)) + _microseconds_from_datetime(self.end)) return data_pb2.TimestampRange(**timestamp_range_kwargs) -class ColumnRange(object): - """A range of columns to restrict to in a row filter. +class TimestampRangeFilter(RowFilter): + """Row filter that limits cells to a range of time. + + :type range_: :class:`TimestampRange` + :param range_: Range of time that cells should match against. + """ + + def __init__(self, range_): + self.range_ = range_ + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.range_ == self.range_ + + def to_pb(self): + """Converts the row filter to a protobuf. + + First converts the ``range_`` on the current object to a protobuf and + then uses it in the ``timestamp_range_filter`` field. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) + + +class ColumnRangeFilter(RowFilter): + """A row filter to restrict to a range of columns. Both the start and end column can be included or excluded in the range. By default, we include them both, but this can be changed with optional @@ -904,29 +875,48 @@ class ColumnRange(object): :type start_column: bytes :param start_column: The start of the range of columns. If no value is - used, it is interpreted as the empty string - (inclusive) by the backend. + used, the backend applies no upper bound to the + values. :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, it - is interpreted as the infinite string (exclusive) by the - backend. + :param end_column: The end of the range of columns. If no value is used, + the backend applies no upper bound to the values. :type inclusive_start: bool :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). + included in the range (or excluded). Defaults + to :data:`True` if ``start_column`` is passed and + no ``inclusive_start`` was given. :type inclusive_end: bool :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). + included in the range (or excluded). Defaults + to :data:`True` if ``end_column`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_column`` is given or if ``inclusive_end`` + is set but no ``end_column`` is given """ def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=True, inclusive_end=True): + inclusive_start=None, inclusive_end=None): self.column_family_id = column_family_id + + if inclusive_start is None: + inclusive_start = True + elif start_column is None: + raise ValueError('Inclusive start was specified but no ' + 'start column was given.') self.start_column = start_column - self.end_column = end_column self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_column is None: + raise ValueError('Inclusive end was specified but no ' + 'end column was given.') + self.end_column = end_column self.inclusive_end = inclusive_end def __eq__(self, other): @@ -938,13 +928,13 @@ def __eq__(self, other): other.inclusive_start == self.inclusive_start and other.inclusive_end == self.inclusive_end) - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): - """Converts the :class:`ColumnRange` to a protobuf. + """Converts the row filter to a protobuf. + + First converts to a :class:`.data_pb2.ColumnRange` and then uses it + in the ``column_range_filter`` field. - :rtype: :class:`.data_pb2.ColumnRange` + :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. """ column_range_kwargs = {'family_name': self.column_family_id} @@ -960,10 +950,42 @@ def to_pb(self): else: key = 'end_qualifier_exclusive' column_range_kwargs[key] = _to_bytes(self.end_column) - return data_pb2.ColumnRange(**column_range_kwargs) + column_range = data_pb2.ColumnRange(**column_range_kwargs) + return data_pb2.RowFilter(column_range_filter=column_range) + + +class ValueRegexFilter(_RegexFilter): + """Row filter for a value regular expression. + + The ``regex`` must be valid RE2 patterns. See Google's + `RE2 reference`_ for the accepted syntax. + + .. _RE2 reference: https://github.com/google/re2/wiki/Syntax + + .. note:: + + Special care need be used with the expression used. Since + each of these properties can contain arbitrary bytes, the ``\\C`` + escape sequence must be used if a true wildcard is desired. The ``.`` + character will not match the new line character ``\\n``, which may be + present in a binary value. -class CellValueRange(object): + :type regex: bytes + :param regex: A regular expression (RE2) to match cells with values that + match this regex. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(value_regex_filter=self.regex) + + +class ValueRangeFilter(RowFilter): """A range of values to restrict to in a row filter. Will only match cells that have values in this range. @@ -973,29 +995,46 @@ class CellValueRange(object): flags. :type start_value: bytes - :param start_value: The start of the range of values. If no value is - used, it is interpreted as the empty string - (inclusive) by the backend. + :param start_value: The start of the range of values. If no value is used, + the backend applies no lower bound to the values. :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, it - is interpreted as the infinite string (exclusive) by the - backend. + :param end_value: The end of the range of values. If no value is used, + the backend applies no upper bound to the values. :type inclusive_start: bool :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). + included in the range (or excluded). Defaults + to :data:`True` if ``start_value`` is passed and + no ``inclusive_start`` was given. :type inclusive_end: bool :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). + included in the range (or excluded). Defaults + to :data:`True` if ``end_value`` is passed and + no ``inclusive_end`` was given. + + :raises: :class:`ValueError ` if ``inclusive_start`` + is set but no ``start_value`` is given or if ``inclusive_end`` + is set but no ``end_value`` is given """ def __init__(self, start_value=None, end_value=None, - inclusive_start=True, inclusive_end=True): + inclusive_start=None, inclusive_end=None): + if inclusive_start is None: + inclusive_start = True + elif start_value is None: + raise ValueError('Inclusive start was specified but no ' + 'start value was given.') self.start_value = start_value - self.end_value = end_value self.inclusive_start = inclusive_start + + if inclusive_end is None: + inclusive_end = True + elif end_value is None: + raise ValueError('Inclusive end was specified but no ' + 'end value was given.') + self.end_value = end_value self.inclusive_end = inclusive_end def __eq__(self, other): @@ -1006,13 +1045,13 @@ def __eq__(self, other): other.inclusive_start == self.inclusive_start and other.inclusive_end == self.inclusive_end) - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): - """Converts the :class:`CellValueRange` to a protobuf. + """Converts the row filter to a protobuf. - :rtype: :class:`.data_pb2.ValueRange` + First converts to a :class:`.data_pb2.ValueRange` and then uses + it to create a row filter protobuf. + + :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. """ value_range_kwargs = {} @@ -1028,10 +1067,135 @@ def to_pb(self): else: key = 'end_value_exclusive' value_range_kwargs[key] = _to_bytes(self.end_value) - return data_pb2.ValueRange(**value_range_kwargs) + value_range = data_pb2.ValueRange(**value_range_kwargs) + return data_pb2.RowFilter(value_range_filter=value_range) + + +class _CellCountFilter(RowFilter): + """Row filter that uses an integer count of cells. + + The cell count is used as an offset or a limit for the number + of results returned. + + :type num_cells: int + :param num_cells: An integer count / offset / limit. + """ + + def __init__(self, num_cells): + self.num_cells = num_cells + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.num_cells == self.num_cells + + +class CellsRowOffsetFilter(_CellCountFilter): + """Row filter to skip cells in a row. + + :type num_cells: int + :param num_cells: Skips the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) + + +class CellsRowLimitFilter(_CellCountFilter): + """Row filter to limit cells in a row. + + :type num_cells: int + :param num_cells: Matches only the first N cells of the row. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) + + +class CellsColumnLimitFilter(_CellCountFilter): + """Row filter to limit cells in a column. + + :type num_cells: int + :param num_cells: Matches only the most recent N cells within each column. + This filters a (family name, column) pair, based on + timestamps of each cell. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) + + +class StripValueTransformerFilter(_BoolFilter): + """Row filter that transforms cells into empty string (0 bytes). + + :type flag: bool + :param flag: If :data:`True`, replaces each cell's value with the empty + string. As the name indicates, this is more useful as a + transformer than a generic query / filter. + """ + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(strip_value_transformer=self.flag) -class RowFilterChain(object): + +class ApplyLabelFilter(RowFilter): + """Filter to apply labels to cells. + + Intended to be used as an intermediate filter on a pre-existing filtered + result set. This was if two sets are combined, the label can tell where + the cell(s) originated.This allows the client to determine which results + were produced from which part of the filter. + + .. note:: + + Due to a technical limitation, it is not currently possible to apply + multiple labels to a cell. + + :type label: str + :param label: Label to apply to cells in the output row. Values must be + at most 15 characters long, and match the pattern + ``[a-z0-9\\-]+``. + """ + + def __init__(self, label): + self.label = label + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + return other.label == self.label + + def to_pb(self): + """Converts the row filter to a protobuf. + + :rtype: :class:`.data_pb2.RowFilter` + :returns: The converted current object. + """ + return data_pb2.RowFilter(apply_label_transformer=self.label) + + +class _FilterCombination(RowFilter): """Chain of row filters. Sends rows through several filters in sequence. The filters are "chained" @@ -1039,12 +1203,12 @@ class RowFilterChain(object): is applied to the filtered output and so on for subsequent filters. :type filters: list - :param filters: List of :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` and/or - :class:`ConditionalRowFilter` + :param filters: List of :class:`RowFilter` """ def __init__(self, filters=None): + if filters is None: + filters = [] self.filters = filters def __eq__(self, other): @@ -1052,11 +1216,20 @@ def __eq__(self, other): return False return other.filters == self.filters - def __ne__(self, other): - return not self.__eq__(other) + +class RowFilterChain(_FilterCombination): + """Chain of row filters. + + Sends rows through several filters in sequence. The filters are "chained" + together to process a row. After the first filter is applied, the second + is applied to the filtered output and so on for subsequent filters. + + :type filters: list + :param filters: List of :class:`RowFilter` + """ def to_pb(self): - """Converts the :class:`RowFilterChain` to a protobuf. + """Converts the row filter to a protobuf. :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. @@ -1066,7 +1239,7 @@ def to_pb(self): return data_pb2.RowFilter(chain=chain) -class RowFilterUnion(object): +class RowFilterUnion(_FilterCombination): """Union of row filters. Sends rows through several filters simultaneously, then @@ -1076,24 +1249,11 @@ class RowFilterUnion(object): they will all appear in the output row in an unspecified mutual order. :type filters: list - :param filters: List of :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` and/or - :class:`ConditionalRowFilter` + :param filters: List of :class:`RowFilter` """ - def __init__(self, filters=None): - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): - """Converts the :class:`RowFilterUnion` to a protobuf. + """Converts the row filter to a protobuf. :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. @@ -1103,8 +1263,8 @@ def to_pb(self): return data_pb2.RowFilter(interleave=interleave) -class ConditionalRowFilter(object): - """Conditional filter +class ConditionalRowFilter(RowFilter): + """Conditional row filter which exhibits ternary behavior. Executes one of two filters based on another filter. If the ``base_filter`` returns any cells in the row, then ``true_filter`` is executed. If not, @@ -1118,20 +1278,16 @@ class ConditionalRowFilter(object): Additionally, executing a :class:`ConditionalRowFilter` has poor performance on the server, especially when ``false_filter`` is set. - :type base_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` + :type base_filter: :class:`RowFilter` :param base_filter: The filter to condition on before executing the true/false filters. - :type true_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` + :type true_filter: :class:`RowFilter` :param true_filter: (Optional) The filter to execute if there are any cells matching ``base_filter``. If not provided, no results will be returned in the true case. - :type false_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or - :class:`ConditionalRowFilter` + :type false_filter: :class:`RowFilter` :param false_filter: (Optional) The filter to execute if there are no cells matching ``base_filter``. If not provided, no results will be returned in the false case. @@ -1149,11 +1305,8 @@ def __eq__(self, other): other.true_filter == self.true_filter and other.false_filter == self.false_filter) - def __ne__(self, other): - return not self.__eq__(other) - def to_pb(self): - """Converts the :class:`ConditionalRowFilter` to a protobuf. + """Converts the row filter to a protobuf. :rtype: :class:`.data_pb2.RowFilter` :returns: The converted current object. diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index d797ecc..0bc6a90 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -448,11 +448,11 @@ def test_commit_with_filter(self): from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter client = _Client() table = _Table(TABLE_NAME, client=client) - row_filter = RowFilter(row_sample_filter=0.33) + row_filter = RowSampleFilter(0.33) row = self._makeOne(ROW_KEY, table, filter_=row_filter) # Create request_pb @@ -693,273 +693,354 @@ def test_it(self): self.assertEqual(expected_output, self._callFUT(sample_input)) -class TestRowFilter(unittest2.TestCase): - - _PROPERTIES = ( - 'sink', - 'pass_all_filter', - 'block_all_filter', - 'row_key_regex_filter', - 'row_sample_filter', - 'family_name_regex_filter', - 'column_qualifier_regex_filter', - 'column_range_filter', - 'timestamp_range_filter', - 'value_regex_filter', - 'value_range_filter', - 'cells_per_row_offset_filter', - 'cells_per_row_limit_filter', - 'cells_per_column_limit_filter', - 'strip_value_transformer', - 'apply_label_transformer', - ) +class Test_BoolFilter(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.row import RowFilter - return RowFilter + from gcloud_bigtable.row import _BoolFilter + return _BoolFilter def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) - def test_constructor_defaults(self): - # Fails unless exactly one property is passed. - with self.assertRaises(TypeError): - self._makeOne() + def test_constructor(self): + flag = object() + row_filter = self._makeOne(flag) + self.assertTrue(row_filter.flag is flag) - def test_constructor_too_many_values(self): - # Fails unless exactly one property is passed. - with self.assertRaises(TypeError): - self._makeOne(row_key_regex_filter=b'value', - cells_per_column_limit_filter=10) - - def _row_filter_values_check(self, row_filter, value_name, value): - for prop_name in self._PROPERTIES: - prop_value = getattr(row_filter, prop_name) - if prop_name == value_name: - self.assertEqual(prop_value, value) - else: - self.assertTrue(prop_value is None) - - def test_constructor_sink(self): - value = True - row_filter = self._makeOne(sink=value) - self._row_filter_values_check(row_filter, 'sink', value) - - def test_constructor_pass_all_filter(self): - value = True - row_filter = self._makeOne(pass_all_filter=value) - self._row_filter_values_check(row_filter, 'pass_all_filter', value) - - def test_constructor_block_all_filter(self): - value = True - row_filter = self._makeOne(block_all_filter=value) - self._row_filter_values_check(row_filter, 'block_all_filter', value) - - def test_constructor_row_key_regex_filter(self): - value = b'row-key-regex' - row_filter = self._makeOne(row_key_regex_filter=value) - self._row_filter_values_check( - row_filter, 'row_key_regex_filter', value) - - def test_constructor_row_sample_filter(self): - value = 0.25 - row_filter = self._makeOne(row_sample_filter=value) - self._row_filter_values_check(row_filter, 'row_sample_filter', value) - - def test_constructor_family_name_regex_filter(self): - value = u'family-regex' - row_filter = self._makeOne(family_name_regex_filter=value) - self._row_filter_values_check( - row_filter, 'family_name_regex_filter', value) - - def test_constructor_column_qualifier_regex_filter(self): - value = b'column-regex' - row_filter = self._makeOne(column_qualifier_regex_filter=value) - self._row_filter_values_check( - row_filter, 'column_qualifier_regex_filter', value) - - def test_constructor_column_range_filter(self): - from gcloud_bigtable.row import ColumnRange - value = ColumnRange(COLUMN_FAMILY_ID) - row_filter = self._makeOne(column_range_filter=value) - self._row_filter_values_check(row_filter, 'column_range_filter', - value) - - def test_constructor_timestamp_range_filter(self): - from gcloud_bigtable.row import TimestampRange - value = TimestampRange() - row_filter = self._makeOne(timestamp_range_filter=value) - self._row_filter_values_check(row_filter, 'timestamp_range_filter', - value) - - def test_constructor_value_regex_filter(self): - value = b'value-regex' - row_filter = self._makeOne(value_regex_filter=value) - self._row_filter_values_check(row_filter, 'value_regex_filter', value) - - def test_constructor_value_range_filter(self): - from gcloud_bigtable.row import CellValueRange - value = CellValueRange() - row_filter = self._makeOne(value_range_filter=value) - self._row_filter_values_check(row_filter, 'value_range_filter', - value) - - def test_constructor_cells_per_row_offset_filter(self): - value = 76 - row_filter = self._makeOne(cells_per_row_offset_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_row_offset_filter', value) - - def test_constructor_cells_per_row_limit_filter(self): - value = 189 - row_filter = self._makeOne(cells_per_row_limit_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_row_limit_filter', value) - - def test_constructor_cells_per_column_limit_filter(self): - value = 10 - row_filter = self._makeOne(cells_per_column_limit_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_column_limit_filter', value) - - def test_constructor_strip_value_transformer(self): - value = True - row_filter = self._makeOne(strip_value_transformer=value) - self._row_filter_values_check(row_filter, 'strip_value_transformer', - value) - - def test_constructor_apply_label_transformer(self): - value = u'label' - row_filter = self._makeOne(apply_label_transformer=value) - self._row_filter_values_check(row_filter, 'apply_label_transformer', - value) + def test___eq__type_differ(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) - def test___eq__(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) - row_filter2 = self._makeOne(strip_value_transformer=True) - # Set every value so we can compare them all. - for prop_name in self._PROPERTIES: - fake_val = object() - setattr(row_filter1, prop_name, fake_val) - setattr(row_filter2, prop_name, fake_val) + def test___eq__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) self.assertEqual(row_filter1, row_filter2) + def test___ne__same_value(self): + flag = object() + row_filter1 = self._makeOne(flag) + row_filter2 = self._makeOne(flag) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestSinkFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import SinkFilter + return SinkFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(sink=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestPassAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import PassAllFilter + return PassAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(pass_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestBlockAllFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import BlockAllFilter + return BlockAllFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(block_all_filter=flag) + self.assertEqual(pb_val, expected_pb) + + +class Test_RegexFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import _RegexFilter + return _RegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + regex = object() + row_filter = self._makeOne(regex) + self.assertTrue(row_filter.regex is regex) + def test___eq__type_differ(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) + regex = object() + row_filter1 = self._makeOne(regex) row_filter2 = object() self.assertNotEqual(row_filter1, row_filter2) + def test___eq__same_value(self): + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) + self.assertEqual(row_filter1, row_filter2) + def test___ne__same_value(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) - row_filter2 = self._makeOne(strip_value_transformer=True) + regex = object() + row_filter1 = self._makeOne(regex) + row_filter2 = self._makeOne(regex) comparison_val = (row_filter1 != row_filter2) self.assertFalse(comparison_val) - def test_to_pb_empty(self): - # Fool the constructor by passing exactly 1 value. - row_filter = self._makeOne(strip_value_transformer=True) - # Make it artificially empty after the fact. - row_filter.strip_value_transformer = None - with self.assertRaises(TypeError): - row_filter.to_pb() +class TestRowKeyRegexFilter(unittest2.TestCase): - def _to_pb_test_helper(self, **kwargs): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + def _getTargetClass(self): + from gcloud_bigtable.row import RowKeyRegexFilter + return RowKeyRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) - row_filter = self._makeOne(**kwargs) + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + regex = b'row-key-regex' + row_filter = self._makeOne(regex) pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(**kwargs) + expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) self.assertEqual(pb_val, expected_pb) - def test_to_pb_with_sink(self): - value = True - self._to_pb_test_helper(sink=value) - def test_to_pb_with_pass_all_filter(self): - value = True - self._to_pb_test_helper(pass_all_filter=value) +class TestRowSampleFilter(unittest2.TestCase): - def test_to_pb_with_block_all_filter(self): - value = True - self._to_pb_test_helper(block_all_filter=value) + def _getTargetClass(self): + from gcloud_bigtable.row import RowSampleFilter + return RowSampleFilter - def test_to_pb_with_row_key_regex_filter(self): - value = b'row-key-regex' - self._to_pb_test_helper(row_key_regex_filter=value) + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) - def test_to_pb_with_row_sample_filter(self): - value = 0.25 - self._to_pb_test_helper(row_sample_filter=value) + def test_constructor(self): + sample = object() + row_filter = self._makeOne(sample) + self.assertTrue(row_filter.sample is sample) - def test_to_pb_with_family_name_regex_filter(self): - value = u'family-regex' - self._to_pb_test_helper(family_name_regex_filter=value) + def test___eq__type_differ(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) - def test_to_pb_with_column_qualifier_regex_filter(self): - value = b'column-regex' - self._to_pb_test_helper(column_qualifier_regex_filter=value) + def test___eq__same_value(self): + sample = object() + row_filter1 = self._makeOne(sample) + row_filter2 = self._makeOne(sample) + self.assertEqual(row_filter1, row_filter2) - def test_to_pb_with_column_range_filter(self): + def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import ColumnRange - - value = ColumnRange(COLUMN_FAMILY_ID) - row_filter = self._makeOne(column_range_filter=value) + sample = 0.25 + row_filter = self._makeOne(sample) pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_range_filter=value.to_pb()) + expected_pb = data_pb2.RowFilter(row_sample_filter=sample) self.assertEqual(pb_val, expected_pb) - def test_to_pb_with_timestamp_range_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import TimestampRange - value = TimestampRange() - row_filter = self._makeOne(timestamp_range_filter=value) +class TestFamilyNameRegexFilter(unittest2.TestCase): + def _getTargetClass(self): + from gcloud_bigtable.row import FamilyNameRegexFilter + return FamilyNameRegexFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = u'family-regex' + row_filter = self._makeOne(regex) pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(timestamp_range_filter=value.to_pb()) + expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) self.assertEqual(pb_val, expected_pb) - def test_to_pb_with_value_regex_filter(self): - value = b'value-regex' - self._to_pb_test_helper(value_regex_filter=value) - def test_to_pb_with_value_range_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import CellValueRange +class TestColumnQualifierRegexFilter(unittest2.TestCase): - value = CellValueRange() - row_filter = self._makeOne(value_range_filter=value) + def _getTargetClass(self): + from gcloud_bigtable.row import ColumnQualifierRegexFilter + return ColumnQualifierRegexFilter + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + regex = b'column-regex' + row_filter = self._makeOne(regex) pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_range_filter=value.to_pb()) + expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) self.assertEqual(pb_val, expected_pb) - def test_to_pb_with_cells_per_row_offset_filter(self): - value = 76 - self._to_pb_test_helper(cells_per_row_offset_filter=value) - def test_to_pb_with_cells_per_row_limit_filter(self): - value = 189 - self._to_pb_test_helper(cells_per_row_limit_filter=value) +class TestColumnRangeFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import ColumnRangeFilter + return ColumnRangeFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) - def test_to_pb_with_cells_per_column_limit_filter(self): - value = 10 - self._to_pb_test_helper(cells_per_column_limit_filter=value) + def test_constructor_defaults(self): + column_family_id = object() + row_filter = self._makeOne(column_family_id) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertEqual(row_filter.start_column, None) + self.assertEqual(row_filter.end_column, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) - def test_to_pb_with_strip_value_transformer(self): - value = True - self._to_pb_test_helper(strip_value_transformer=value) + def test_constructor_explicit(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter = self._makeOne(column_family_id, start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.column_family_id is column_family_id) + self.assertTrue(row_filter.start_column is start_column) + self.assertTrue(row_filter.end_column is end_column) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) - def test_to_pb_with_apply_label_transformer(self): - value = u'label' - self._to_pb_test_helper(apply_label_transformer=value) + def test_constructor_bad_start(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_start=True) + + def test_constructor_bad_end(self): + column_family_id = object() + self.assertRaises(ValueError, self._makeOne, + column_family_id, inclusive_end=True) + + def test___eq__(self): + column_family_id = object() + start_column = object() + end_column = object() + inclusive_start = object() + inclusive_end = object() + row_filter1 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(column_family_id, + start_column=start_column, + end_column=end_column, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) + + def test___eq__type_differ(self): + column_family_id = object() + row_filter1 = self._makeOne(column_family_id) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + row_filter = self._makeOne(column_family_id) + col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_start(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_start(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, start_column=column, + inclusive_start=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + start_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_inclusive_end(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_inclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) + + def test_to_pb_exclusive_end(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + column_family_id = u'column-family-id' + column = b'column' + row_filter = self._makeOne(column_family_id, end_column=column, + inclusive_end=False) + col_range_pb = data_pb2.ColumnRange( + family_name=column_family_id, + end_qualifier_exclusive=column, + ) + expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) class TestTimestampRange(unittest2.TestCase): @@ -1027,238 +1108,339 @@ def test_to_pb(self): self._to_pb_helper(start_micros=start_micros, end_micros=end_micros) - def test_to_pb_just_start(self): + def test_to_pb_start_only(self): # Makes sure already milliseconds granularity start_micros = 30871000 self._to_pb_helper(start_micros=start_micros) - def test_to_pb_just_end(self): + def test_to_pb_end_only(self): # Makes sure already milliseconds granularity end_micros = 12939371000 self._to_pb_helper(end_micros=end_micros) -class TestColumnRange(unittest2.TestCase): +class TestTimestampRangeFilter(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.row import ColumnRange - return ColumnRange + from gcloud_bigtable.row import TimestampRangeFilter + return TimestampRangeFilter def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) - def test_constructor_defaults(self): - column_family_id = object() - col_range = self._makeOne(column_family_id) - self.assertTrue(col_range.column_family_id is column_family_id) - self.assertEqual(col_range.start_column, None) - self.assertEqual(col_range.end_column, None) - self.assertTrue(col_range.inclusive_start) - self.assertTrue(col_range.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - col_range = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(col_range.column_family_id is column_family_id) - self.assertTrue(col_range.start_column is start_column) - self.assertTrue(col_range.end_column is end_column) - self.assertTrue(col_range.inclusive_start is inclusive_start) - self.assertTrue(col_range.inclusive_end is inclusive_end) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - col_range1 = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - col_range2 = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(col_range1, col_range2) + def test_constructor(self): + range_ = object() + row_filter = self._makeOne(range_) + self.assertTrue(row_filter.range_ is range_) def test___eq__type_differ(self): - column_family_id = object() - col_range1 = self._makeOne(column_family_id) - col_range2 = object() - self.assertNotEqual(col_range1, col_range2) + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) - def test___ne__same_value(self): - column_family_id = object() - col_range1 = self._makeOne(column_family_id) - col_range2 = self._makeOne(column_family_id) - comparison_val = (col_range1 != col_range2) - self.assertFalse(comparison_val) + def test___eq__same_value(self): + range_ = object() + row_filter1 = self._makeOne(range_) + row_filter2 = self._makeOne(range_) + self.assertEqual(row_filter1, row_filter2) def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + from gcloud_bigtable.row import TimestampRange - col_range = self._makeOne(COLUMN_FAMILY_ID) - expected_pb = data_pb2.ColumnRange(family_name=COLUMN_FAMILY_ID) - self.assertEqual(col_range.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID, start_column=COLUMN) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - start_qualifier_inclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) + range_ = TimestampRange() + row_filter = self._makeOne(range_) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + timestamp_range_filter=data_pb2.TimestampRange()) + self.assertEqual(pb_val, expected_pb) - def test_to_pb_exclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - col_range = self._makeOne(COLUMN_FAMILY_ID, start_column=COLUMN, - inclusive_start=False) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - start_qualifier_exclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) +class TestValueRegexFilter(unittest2.TestCase): - def test_to_pb_inclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + def _getTargetClass(self): + from gcloud_bigtable.row import ValueRegexFilter + return ValueRegexFilter - col_range = self._makeOne(COLUMN_FAMILY_ID, end_column=COLUMN) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - end_qualifier_inclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) - def test_to_pb_exclusive_end(self): + def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - col_range = self._makeOne(COLUMN_FAMILY_ID, end_column=COLUMN, - inclusive_end=False) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - end_qualifier_exclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) + regex = b'value-regex' + row_filter = self._makeOne(regex) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(value_regex_filter=regex) + self.assertEqual(pb_val, expected_pb) -class TestCellValueRange(unittest2.TestCase): +class TestValueRangeFilter(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.row import CellValueRange - return CellValueRange + from gcloud_bigtable.row import ValueRangeFilter + return ValueRangeFilter def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def test_constructor_defaults(self): - val_range = self._makeOne() - self.assertEqual(val_range.start_value, None) - self.assertEqual(val_range.end_value, None) - self.assertTrue(val_range.inclusive_start) - self.assertTrue(val_range.inclusive_end) + row_filter = self._makeOne() + self.assertEqual(row_filter.start_value, None) + self.assertEqual(row_filter.end_value, None) + self.assertTrue(row_filter.inclusive_start) + self.assertTrue(row_filter.inclusive_end) def test_constructor_explicit(self): start_value = object() end_value = object() inclusive_start = object() inclusive_end = object() - val_range = self._makeOne(start_value=start_value, end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(val_range.start_value is start_value) - self.assertTrue(val_range.end_value is end_value) - self.assertTrue(val_range.inclusive_start is inclusive_start) - self.assertTrue(val_range.inclusive_end is inclusive_end) + row_filter = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertTrue(row_filter.start_value is start_value) + self.assertTrue(row_filter.end_value is end_value) + self.assertTrue(row_filter.inclusive_start is inclusive_start) + self.assertTrue(row_filter.inclusive_end is inclusive_end) + + def test_constructor_bad_start(self): + self.assertRaises(ValueError, self._makeOne, inclusive_start=True) + + def test_constructor_bad_end(self): + self.assertRaises(ValueError, self._makeOne, inclusive_end=True) def test___eq__(self): start_value = object() end_value = object() inclusive_start = object() inclusive_end = object() - val_range1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - val_range2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(val_range1, val_range2) + row_filter1 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + row_filter2 = self._makeOne(start_value=start_value, + end_value=end_value, + inclusive_start=inclusive_start, + inclusive_end=inclusive_end) + self.assertEqual(row_filter1, row_filter2) def test___eq__type_differ(self): - val_range1 = self._makeOne() - val_range2 = object() - self.assertNotEqual(val_range1, val_range2) - - def test___ne__same_value(self): - val_range1 = self._makeOne() - val_range2 = self._makeOne() - comparison_val = (val_range1 != val_range2) - self.assertFalse(comparison_val) + row_filter1 = self._makeOne() + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - val_range = self._makeOne() - expected_pb = data_pb2.ValueRange() - self.assertEqual(val_range.to_pb(), expected_pb) + row_filter = self._makeOne() + expected_pb = data_pb2.RowFilter( + value_range_filter=data_pb2.ValueRange()) + self.assertEqual(row_filter.to_pb(), expected_pb) def test_to_pb_inclusive_start(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 value = b'some-value' - val_range = self._makeOne(start_value=value) - expected_pb = data_pb2.ValueRange(start_value_inclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) + row_filter = self._makeOne(start_value=value) + val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) def test_to_pb_exclusive_start(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 value = b'some-value' - val_range = self._makeOne(start_value=value, inclusive_start=False) - expected_pb = data_pb2.ValueRange(start_value_exclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) + row_filter = self._makeOne(start_value=value, inclusive_start=False) + val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) def test_to_pb_inclusive_end(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 value = b'some-value' - val_range = self._makeOne(end_value=value) - expected_pb = data_pb2.ValueRange(end_value_inclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) + row_filter = self._makeOne(end_value=value) + val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) def test_to_pb_exclusive_end(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 value = b'some-value' - val_range = self._makeOne(end_value=value, inclusive_end=False) - expected_pb = data_pb2.ValueRange(end_value_exclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) + row_filter = self._makeOne(end_value=value, inclusive_end=False) + val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) + expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) + self.assertEqual(row_filter.to_pb(), expected_pb) -class TestRowFilterChain(unittest2.TestCase): +class Test_CellCountFilter(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable.row import RowFilterChain - return RowFilterChain + from gcloud_bigtable.row import _CellCountFilter + return _CellCountFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + num_cells = object() + row_filter = self._makeOne(num_cells) + self.assertTrue(row_filter.num_cells is num_cells) + + def test___eq__type_differ(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + self.assertEqual(row_filter1, row_filter2) + + def test___ne__same_value(self): + num_cells = object() + row_filter1 = self._makeOne(num_cells) + row_filter2 = self._makeOne(num_cells) + comparison_val = (row_filter1 != row_filter2) + self.assertFalse(comparison_val) + + +class TestCellsRowOffsetFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import CellsRowOffsetFilter + return CellsRowOffsetFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 76 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsRowLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import CellsRowLimitFilter + return CellsRowLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 189 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestCellsColumnLimitFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import CellsColumnLimitFilter + return CellsColumnLimitFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + num_cells = 10 + row_filter = self._makeOne(num_cells) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter( + cells_per_column_limit_filter=num_cells) + self.assertEqual(pb_val, expected_pb) + + +class TestStripValueTransformerFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import StripValueTransformerFilter + return StripValueTransformerFilter + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + flag = True + row_filter = self._makeOne(flag) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) + self.assertEqual(pb_val, expected_pb) + + +class TestApplyLabelFilter(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import ApplyLabelFilter + return ApplyLabelFilter def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) def test_constructor(self): + label = object() + row_filter = self._makeOne(label) + self.assertTrue(row_filter.label is label) + + def test___eq__type_differ(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = object() + self.assertNotEqual(row_filter1, row_filter2) + + def test___eq__same_value(self): + label = object() + row_filter1 = self._makeOne(label) + row_filter2 = self._makeOne(label) + self.assertEqual(row_filter1, row_filter2) + + def test_to_pb(self): + from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 + + label = u'label' + row_filter = self._makeOne(label) + pb_val = row_filter.to_pb() + expected_pb = data_pb2.RowFilter(apply_label_transformer=label) + self.assertEqual(pb_val, expected_pb) + + +class Test_FilterCombination(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import _FilterCombination + return _FilterCombination + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor_defaults(self): + row_filter = self._makeOne() + self.assertEqual(row_filter.filters, []) + + def test_constructor_explicit(self): filters = object() - filter_chain = self._makeOne(filters=filters) - self.assertTrue(filter_chain.filters is filters) + row_filter = self._makeOne(filters=filters) + self.assertTrue(row_filter.filters is filters) def test___eq__(self): filters = object() @@ -1272,21 +1454,25 @@ def test___eq__type_differ(self): row_filter2 = object() self.assertNotEqual(row_filter1, row_filter2) - def test___ne__same_value(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) + +class TestRowFilterChain(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.row import RowFilterChain + return RowFilterChain + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) + row_filter1 = StripValueTransformerFilter(True) row_filter1_pb = row_filter1.to_pb() - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter2 = RowSampleFilter(0.25) row_filter2_pb = row_filter2.to_pb() row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) @@ -1301,15 +1487,17 @@ def test_to_pb(self): def test_to_pb_nested(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsRowLimitFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) row_filter3_pb = row_filter3.to_pb() - row_filter4 = RowFilter(cells_per_row_limit_filter=11) + row_filter4 = CellsRowLimitFilter(11) row_filter4_pb = row_filter4.to_pb() row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) @@ -1332,38 +1520,15 @@ def _getTargetClass(self): def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) - def test_constructor(self): - filters = object() - filter_union = self._makeOne(filters=filters) - self.assertTrue(filter_union.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) + row_filter1 = StripValueTransformerFilter(True) row_filter1_pb = row_filter1.to_pb() - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter2 = RowSampleFilter(0.25) row_filter2_pb = row_filter2.to_pb() row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) @@ -1378,15 +1543,17 @@ def test_to_pb(self): def test_to_pb_nested(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsRowLimitFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter1 = StripValueTransformerFilter(True) + row_filter2 = RowSampleFilter(0.25) row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) row_filter3_pb = row_filter3.to_pb() - row_filter4 = RowFilter(cells_per_row_limit_filter=11) + row_filter4 = CellsRowLimitFilter(11) row_filter4_pb = row_filter4.to_pb() row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) @@ -1442,30 +1609,19 @@ def test___eq__type_differ(self): cond_filter2 = object() self.assertNotEqual(cond_filter1, cond_filter2) - def test___ne__same_value(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - comparison_val = (cond_filter1 != cond_filter2) - self.assertFalse(comparison_val) - def test_to_pb(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import CellsRowOffsetFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) + row_filter1 = StripValueTransformerFilter(True) row_filter1_pb = row_filter1.to_pb() - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter2 = RowSampleFilter(0.25) row_filter2_pb = row_filter2.to_pb() - row_filter3 = RowFilter(cells_per_row_limit_filter=11) + row_filter3 = CellsRowOffsetFilter(11) row_filter3_pb = row_filter3.to_pb() row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, @@ -1483,12 +1639,13 @@ def test_to_pb(self): def test_to_pb_true_only(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) + row_filter1 = StripValueTransformerFilter(True) row_filter1_pb = row_filter1.to_pb() - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter2 = RowSampleFilter(0.25) row_filter2_pb = row_filter2.to_pb() row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) @@ -1504,12 +1661,13 @@ def test_to_pb_true_only(self): def test_to_pb_false_only(self): from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter + from gcloud_bigtable.row import StripValueTransformerFilter - row_filter1 = RowFilter(strip_value_transformer=True) + row_filter1 = StripValueTransformerFilter(True) row_filter1_pb = row_filter1.to_pb() - row_filter2 = RowFilter(row_sample_filter=0.25) + row_filter2 = RowSampleFilter(0.25) row_filter2_pb = row_filter2.to_pb() row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py index 17781c6..9a430f2 100644 --- a/gcloud_bigtable/test_table.py +++ b/gcloud_bigtable/test_table.py @@ -517,10 +517,10 @@ def test_row_range_both_keys(self): def test_with_filter(self): from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row import RowFilter + from gcloud_bigtable.row import RowSampleFilter table_name = 'table_name' - row_filter = RowFilter(row_sample_filter=0.33) + row_filter = RowSampleFilter(0.33) result = self._callFUT(table_name, filter_=row_filter) expected_result = messages_pb2.ReadRowsRequest( table_name=table_name, diff --git a/pylintrc_default b/pylintrc_default index 79108b1..912d349 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -205,7 +205,7 @@ no-space-check = # Maximum number of lines in a module # DEFAULT: max-module-lines=1000 # RATIONALE: API-mapping -max-module-lines=1546 +max-module-lines=1764 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). From 4d9e110731127d7b3a57e305366daff7d9578b07 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 20:31:04 -0800 Subject: [PATCH 40/65] Update system tests for new RowFilters. --- system_tests/run.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/system_tests/run.py b/system_tests/run.py index a0f7985..f95f164 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -28,7 +28,8 @@ from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable.client import Client from gcloud_bigtable.column_family import MaxVersionsGCRule -from gcloud_bigtable.row import RowFilter +from gcloud_bigtable.row import ApplyLabelFilter +from gcloud_bigtable.row import ColumnQualifierRegexFilter from gcloud_bigtable.row import RowFilterChain from gcloud_bigtable.row import RowFilterUnion from gcloud_bigtable.row_data import Cell @@ -418,14 +419,14 @@ def test_read_with_label_applied(self): # Combine a label with column 1. label1 = u'label-red' - label1_filter = RowFilter(apply_label_transformer=label1) - col1_filter = RowFilter(column_qualifier_regex_filter=COL_NAME1) + label1_filter = ApplyLabelFilter(label1) + col1_filter = ColumnQualifierRegexFilter(COL_NAME1) chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) # Combine a label with column 2. label2 = u'label-blue' - label2_filter = RowFilter(apply_label_transformer=label2) - col2_filter = RowFilter(column_qualifier_regex_filter=COL_NAME2) + label2_filter = ApplyLabelFilter(label2) + col2_filter = ColumnQualifierRegexFilter(COL_NAME2) chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) # Bring our two labeled columns together. From 959cbfa37fc52e84fb087f376ba17eea40750941 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 21:35:09 -0800 Subject: [PATCH 41/65] Updating Makefile to only use gRPC when needed. --- Makefile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1254d35..9caba1d 100644 --- a/Makefile +++ b/Makefile @@ -15,21 +15,31 @@ generate: [ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client cd cloud-bigtable-client && git pull origin master mkdir -p $(GENERATED_DIR) - # Generate all *_pb2.py files. + # Generate all *_pb2.py files that require gRPC. $(PROTOC_CMD) \ --proto_path=$(PROTOS_DIR) \ --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ --python_out=$(GENERATED_DIR) \ --grpc_out=$(GENERATED_DIR) \ - $(PROTOS_DIR)/google/bigtable/v1/*.proto \ - $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto \ + $(PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \ + $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \ + $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto \ + $(PROTOS_DIR)/google/longrunning/operations.proto + # Generate all *_pb2.py files that do not require gRPC. + $(PROTOC_CMD) \ + --proto_path=$(PROTOS_DIR) \ + --python_out=$(GENERATED_DIR) \ + $(PROTOS_DIR)/google/bigtable/v1/bigtable_data.proto \ + $(PROTOS_DIR)/google/bigtable/v1/bigtable_service_messages.proto \ + $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto \ + $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \ + $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \ + $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto \ $(PROTOS_DIR)/google/api/*.proto \ $(PROTOS_DIR)/google/protobuf/any.proto \ $(PROTOS_DIR)/google/protobuf/duration.proto \ $(PROTOS_DIR)/google/protobuf/empty.proto \ $(PROTOS_DIR)/google/protobuf/timestamp.proto \ - $(PROTOS_DIR)/google/longrunning/operations.proto \ $(PROTOS_DIR)/google/rpc/status.proto # Move the newly generated *_pb2.py files into our library. mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) From 5fe0325286ff915ff9fc2a6b027be061b7126d5c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 22 Dec 2015 21:37:18 -0800 Subject: [PATCH 42/65] Updating pb2 files that do not require gRPC. --- gcloud_bigtable/_generated/annotations_pb2.py | 6 ------ gcloud_bigtable/_generated/any_pb2.py | 6 ------ gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py | 6 ------ .../_generated/bigtable_cluster_service_messages_pb2.py | 6 ------ gcloud_bigtable/_generated/bigtable_data_pb2.py | 6 ------ gcloud_bigtable/_generated/bigtable_service_messages_pb2.py | 6 ------ gcloud_bigtable/_generated/bigtable_table_data_pb2.py | 6 ------ .../_generated/bigtable_table_service_messages_pb2.py | 6 ------ gcloud_bigtable/_generated/duration_pb2.py | 6 ------ gcloud_bigtable/_generated/empty_pb2.py | 6 ------ gcloud_bigtable/_generated/http_pb2.py | 6 ------ gcloud_bigtable/_generated/status_pb2.py | 6 ------ gcloud_bigtable/_generated/timestamp_pb2.py | 6 ------ 13 files changed, 78 deletions(-) diff --git a/gcloud_bigtable/_generated/annotations_pb2.py b/gcloud_bigtable/_generated/annotations_pb2.py index 8f98861..905b53c 100644 --- a/gcloud_bigtable/_generated/annotations_pb2.py +++ b/gcloud_bigtable/_generated/annotations_pb2.py @@ -41,10 +41,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\020AnnotationsProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/any_pb2.py b/gcloud_bigtable/_generated/any_pb2.py index 862ab1e..4712c50 100644 --- a/gcloud_bigtable/_generated/any_pb2.py +++ b/gcloud_bigtable/_generated/any_pb2.py @@ -73,10 +73,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\010AnyProtoP\001\240\001\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py index 6be8475..a01cd63 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py @@ -221,10 +221,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n$com.google.bigtable.admin.cluster.v1B\030BigtableClusterDataProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py index 7fbcbd1..0b519bd 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py @@ -535,10 +535,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n$com.google.bigtable.admin.cluster.v1B#BigtableClusterServiceMessagesProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_data_pb2.py b/gcloud_bigtable/_generated/bigtable_data_pb2.py index 58a3b7b..e3183a4 100644 --- a/gcloud_bigtable/_generated/bigtable_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_data_pb2.py @@ -1176,10 +1176,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\021BigtableDataProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py index 9b5b5d4..e676bef 100644 --- a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py @@ -529,10 +529,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\034BigtableServiceMessagesProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py index 5cbc242..7c36fe9 100644 --- a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py @@ -374,10 +374,4 @@ DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\026BigtableTableDataProtoP\001') _TABLE_COLUMNFAMILIESENTRY.has_options = True _TABLE_COLUMNFAMILIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), b'8\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py index 245a0f0..f79cc23 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py @@ -386,10 +386,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/duration_pb2.py b/gcloud_bigtable/_generated/duration_pb2.py index ddb81a7..12fd411 100644 --- a/gcloud_bigtable/_generated/duration_pb2.py +++ b/gcloud_bigtable/_generated/duration_pb2.py @@ -73,10 +73,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\rDurationProtoP\001\240\001\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/empty_pb2.py b/gcloud_bigtable/_generated/empty_pb2.py index 701a9a6..5dfc0ae 100644 --- a/gcloud_bigtable/_generated/empty_pb2.py +++ b/gcloud_bigtable/_generated/empty_pb2.py @@ -59,10 +59,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\nEmptyProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/http_pb2.py b/gcloud_bigtable/_generated/http_pb2.py index 44885b8..a70ba4d 100644 --- a/gcloud_bigtable/_generated/http_pb2.py +++ b/gcloud_bigtable/_generated/http_pb2.py @@ -184,10 +184,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\tHttpProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/status_pb2.py b/gcloud_bigtable/_generated/status_pb2.py index a715af1..1883dcc 100644 --- a/gcloud_bigtable/_generated/status_pb2.py +++ b/gcloud_bigtable/_generated/status_pb2.py @@ -83,10 +83,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.rpcB\013StatusProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/timestamp_pb2.py b/gcloud_bigtable/_generated/timestamp_pb2.py index 66a6cf6..cc1ee18 100644 --- a/gcloud_bigtable/_generated/timestamp_pb2.py +++ b/gcloud_bigtable/_generated/timestamp_pb2.py @@ -73,10 +73,4 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\016TimestampProtoP\001\240\001\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities # @@protoc_insertion_point(module_scope) From 3c5b0591f04597ecbb854a1365120fab8fb4dc59 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 6 Jan 2016 16:16:52 -0800 Subject: [PATCH 43/65] Ditching auxiliary protos for googleapis-common-protos. --- Makefile | 29 +- gcloud_bigtable/_generated/_annotations.proto | 29 - gcloud_bigtable/_generated/_any.proto | 87 --- gcloud_bigtable/_generated/_duration.proto | 78 --- gcloud_bigtable/_generated/_empty.proto | 34 -- gcloud_bigtable/_generated/_http.proto | 127 ---- gcloud_bigtable/_generated/_operations.proto | 144 ----- gcloud_bigtable/_generated/_status.proto | 88 --- gcloud_bigtable/_generated/_timestamp.proto | 89 --- gcloud_bigtable/_generated/annotations_pb2.py | 44 -- gcloud_bigtable/_generated/any_pb2.py | 76 --- .../_generated/bigtable_cluster_data_pb2.py | 6 +- .../bigtable_cluster_service_messages_pb2.py | 2 +- .../bigtable_cluster_service_pb2.py | 38 +- .../_generated/bigtable_service_pb2.py | 20 +- .../_generated/bigtable_table_data_pb2.py | 4 +- .../_generated/bigtable_table_service_pb2.py | 52 +- gcloud_bigtable/_generated/duration_pb2.py | 76 --- gcloud_bigtable/_generated/empty_pb2.py | 62 -- gcloud_bigtable/_generated/http_pb2.py | 187 ------ .../_generated/operations_grpc_pb2.py | 205 +++++++ gcloud_bigtable/_generated/operations_pb2.py | 543 ------------------ gcloud_bigtable/_generated/status_pb2.py | 86 --- gcloud_bigtable/_generated/timestamp_pb2.py | 76 --- gcloud_bigtable/client.py | 4 +- gcloud_bigtable/cluster.py | 17 +- gcloud_bigtable/column_family.py | 3 +- gcloud_bigtable/row.py | 2 +- gcloud_bigtable/table.py | 4 +- gcloud_bigtable/test_cluster.py | 20 +- gcloud_bigtable/test_column_family.py | 20 +- gcloud_bigtable/test_row.py | 2 +- gcloud_bigtable/test_table.py | 4 +- pylintrc_default | 8 - scripts/make_operations_grpc.py | 116 ++++ scripts/rewrite_imports.py | 39 +- setup.py | 3 +- 37 files changed, 432 insertions(+), 1992 deletions(-) delete mode 100644 gcloud_bigtable/_generated/_annotations.proto delete mode 100644 gcloud_bigtable/_generated/_any.proto delete mode 100644 gcloud_bigtable/_generated/_duration.proto delete mode 100644 gcloud_bigtable/_generated/_empty.proto delete mode 100644 gcloud_bigtable/_generated/_http.proto delete mode 100644 gcloud_bigtable/_generated/_operations.proto delete mode 100644 gcloud_bigtable/_generated/_status.proto delete mode 100644 gcloud_bigtable/_generated/_timestamp.proto delete mode 100644 gcloud_bigtable/_generated/annotations_pb2.py delete mode 100644 gcloud_bigtable/_generated/any_pb2.py delete mode 100644 gcloud_bigtable/_generated/duration_pb2.py delete mode 100644 gcloud_bigtable/_generated/empty_pb2.py delete mode 100644 gcloud_bigtable/_generated/http_pb2.py create mode 100644 gcloud_bigtable/_generated/operations_grpc_pb2.py delete mode 100644 gcloud_bigtable/_generated/operations_pb2.py delete mode 100644 gcloud_bigtable/_generated/status_pb2.py delete mode 100644 gcloud_bigtable/_generated/timestamp_pb2.py create mode 100644 scripts/make_operations_grpc.py diff --git a/Makefile b/Makefile index 9caba1d..c25b6fc 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,7 @@ generate: --grpc_out=$(GENERATED_DIR) \ $(PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \ $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto \ - $(PROTOS_DIR)/google/longrunning/operations.proto + $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto # Generate all *_pb2.py files that do not require gRPC. $(PROTOC_CMD) \ --proto_path=$(PROTOS_DIR) \ @@ -34,37 +33,17 @@ generate: $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto \ $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \ $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto \ - $(PROTOS_DIR)/google/api/*.proto \ - $(PROTOS_DIR)/google/protobuf/any.proto \ - $(PROTOS_DIR)/google/protobuf/duration.proto \ - $(PROTOS_DIR)/google/protobuf/empty.proto \ - $(PROTOS_DIR)/google/protobuf/timestamp.proto \ - $(PROTOS_DIR)/google/rpc/status.proto + $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto # Move the newly generated *_pb2.py files into our library. mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(FINAL_DIR) mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(FINAL_DIR) - mv $(GENERATED_DIR)/google/api/* $(FINAL_DIR) - mv $(GENERATED_DIR)/google/protobuf/any_pb2.py $(FINAL_DIR) - mv $(GENERATED_DIR)/google/protobuf/duration_pb2.py $(FINAL_DIR) - mv $(GENERATED_DIR)/google/protobuf/empty_pb2.py $(FINAL_DIR) - mv $(GENERATED_DIR)/google/protobuf/timestamp_pb2.py $(FINAL_DIR) - mv $(GENERATED_DIR)/google/longrunning/operations_pb2.py $(FINAL_DIR) - mv $(GENERATED_DIR)/google/rpc/status_pb2.py $(FINAL_DIR) # Remove all existing *.proto files before we replace rm -f $(FINAL_DIR)/*.proto # Copy over the *.proto files into our library. cp $(PROTOS_DIR)/google/bigtable/v1/*.proto $(FINAL_DIR) cp $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(FINAL_DIR) cp $(PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/api/*.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/protobuf/any.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/protobuf/duration.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/protobuf/empty.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/protobuf/timestamp.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/longrunning/operations.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/rpc/status.proto $(FINAL_DIR) # Rename all *.proto files in our library with an # underscore and remove executable bit. cd $(FINAL_DIR) && \ @@ -72,6 +51,10 @@ generate: chmod -x $$filename ; \ mv $$filename _$$filename ; \ done + # Separate the gRPC parts of the operations service from the + # non-gRPC parts so that the protos from `googleapis-common-protos` + # can be used without gRPC. + python scripts/make_operations_grpc.py # Rewrite the imports in the generated *_pb2.py files. python scripts/rewrite_imports.py diff --git a/gcloud_bigtable/_generated/_annotations.proto b/gcloud_bigtable/_generated/_annotations.proto deleted file mode 100644 index cbd18b8..0000000 --- a/gcloud_bigtable/_generated/_annotations.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.api; - -import "google/api/http.proto"; -import "google/protobuf/descriptor.proto"; - -option java_multiple_files = true; -option java_outer_classname = "AnnotationsProto"; -option java_package = "com.google.api"; - -extend google.protobuf.MethodOptions { - // See `HttpRule`. - HttpRule http = 72295728; -} diff --git a/gcloud_bigtable/_generated/_any.proto b/gcloud_bigtable/_generated/_any.proto deleted file mode 100644 index 7c79e57..0000000 --- a/gcloud_bigtable/_generated/_any.proto +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.protobuf; - -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "AnyProto"; -option java_package = "com.google.protobuf"; - - -// `Any` contains an arbitrary serialized message along with a URL -// that describes the type of the serialized message. -// -// The proto runtimes and/or compiler will eventually -// provide utilities to pack/unpack Any values (projected Q1/15). -// -// # JSON -// The JSON representation of an `Any` value uses the regular -// representation of the deserialized, embedded message, with an -// additional field `@type` which contains the type URL. Example: -// -// package google.profile; -// message Person { -// string first_name = 1; -// string last_name = 2; -// } -// -// { -// "@type": "type.googleapis.com/google.profile.Person", -// "firstName": , -// "lastName": -// } -// -// If the embedded message type is well-known and has a custom JSON -// representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the the `@type` -// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): -// -// { -// "@type": "type.googleapis.com/google.protobuf.Duration", -// "value": "1.212s" -// } -// -message Any { - // A URL/resource name whose content describes the type of the - // serialized message. - // - // For URLs which use the schema `http`, `https`, or no schema, the - // following restrictions and interpretations apply: - // - // * If no schema is provided, `https` is assumed. - // * The last segment of the URL's path must represent the fully - // qualified name of the type (as in `path/google.protobuf.Duration`). - // * An HTTP GET on the URL must yield a [google.protobuf.Type][google.protobuf.Type] - // value in binary format, or produce an error. - // * Applications are allowed to cache lookup results based on the - // URL, or have them precompiled into a binary to avoid any - // lookup. Therefore, binary compatibility needs to be preserved - // on changes to types. (Use versioned type names to manage - // breaking changes.) - // - // Schemas other than `http`, `https` (or the empty schema) might be - // used with implementation specific semantics. - // - // Types originating from the `google.*` package - // namespace should use `type.googleapis.com/full.type.name` (without - // schema and path). A type service will eventually become available which - // serves those URLs (projected Q2/15). - string type_url = 1; - - // Must be valid serialized data of the above specified type. - bytes value = 2; -} diff --git a/gcloud_bigtable/_generated/_duration.proto b/gcloud_bigtable/_generated/_duration.proto deleted file mode 100644 index 15e9d44..0000000 --- a/gcloud_bigtable/_generated/_duration.proto +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.protobuf; - -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "DurationProto"; -option java_package = "com.google.protobuf"; - - -// A Duration represents a signed, fixed-length span of time represented -// as a count of seconds and fractions of seconds at nanosecond -// resolution. It is independent of any calendar and concepts like "day" -// or "month". It is related to Timestamp in that the difference between -// two Timestamp values is a Duration and it can be added or subtracted -// from a Timestamp. Range is approximately +-10,000 years. -// -// Example 1: Compute Duration from two Timestamps in pseudo code. -// -// Timestamp start = ...; -// Timestamp end = ...; -// Duration duration = ...; -// -// duration.seconds = end.seconds - start.seconds; -// duration.nanos = end.nanos - start.nanos; -// -// if (duration.seconds < 0 && duration.nanos > 0) { -// duration.seconds += 1; -// duration.nanos -= 1000000000; -// } else if (durations.seconds > 0 && duration.nanos < 0) { -// duration.seconds -= 1; -// duration.nanos += 1000000000; -// } -// -// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. -// -// Timestamp start = ...; -// Duration duration = ...; -// Timestamp end = ...; -// -// end.seconds = start.seconds + duration.seconds; -// end.nanos = start.nanos + duration.nanos; -// -// if (end.nanos < 0) { -// end.seconds -= 1; -// end.nanos += 1000000000; -// } else if (end.nanos >= 1000000000) { -// end.seconds += 1; -// end.nanos -= 1000000000; -// } -// -message Duration { - // Signed seconds of the span of time. Must be from -315,576,000,000 - // to +315,576,000,000 inclusive. - int64 seconds = 1; - - // Signed fractions of a second at nanosecond resolution of the span - // of time. Durations less than one second are represented with a 0 - // `seconds` field and a positive or negative `nanos` field. For durations - // of one second or more, a non-zero value for the `nanos` field must be - // of the same sign as the `seconds` field. Must be from -999,999,999 - // to +999,999,999 inclusive. - int32 nanos = 2; -} diff --git a/gcloud_bigtable/_generated/_empty.proto b/gcloud_bigtable/_generated/_empty.proto deleted file mode 100644 index 43b06e8..0000000 --- a/gcloud_bigtable/_generated/_empty.proto +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.protobuf; - -option java_multiple_files = true; -option java_outer_classname = "EmptyProto"; -option java_package = "com.google.protobuf"; - - -// A generic empty message that you can re-use to avoid defining duplicated -// empty messages in your APIs. A typical example is to use it as the request -// or the response type of an API method. For instance: -// -// service Foo { -// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); -// } -// -message Empty { - -} diff --git a/gcloud_bigtable/_generated/_http.proto b/gcloud_bigtable/_generated/_http.proto deleted file mode 100644 index ce07aa1..0000000 --- a/gcloud_bigtable/_generated/_http.proto +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.api; - -option java_multiple_files = true; -option java_outer_classname = "HttpProto"; -option java_package = "com.google.api"; - - -// `HttpRule` defines the mapping of an RPC method to one or more HTTP REST API -// methods. The mapping determines what portions of the request message are -// populated from the path, query parameters, or body of the HTTP request. The -// mapping is typically specified as an `google.api.http` annotation, see -// "google/api/annotations.proto" for details. -// -// The mapping consists of a mandatory field specifying a path template and an -// optional `body` field specifying what data is represented in the HTTP request -// body. The field name for the path indicates the HTTP method. Example: -// -// ``` -// package google.storage.v2; -// -// import "google/api/annotations.proto"; -// -// service Storage { -// rpc CreateObject(CreateObjectRequest) returns (Object) { -// option (google.api.http) { -// post: "/v2/{bucket_name=buckets/*}/objects" -// body: "object" -// }; -// }; -// } -// ``` -// -// Here `bucket_name` and `object` bind to fields of the request message -// `CreateObjectRequest`. -// -// The rules for mapping HTTP path, query parameters, and body fields -// to the request message are as follows: -// -// 1. The `body` field specifies either `*` or a field path, or is -// omitted. If omitted, it assumes there is no HTTP body. -// 2. Leaf fields (recursive expansion of nested messages in the -// request) can be classified into three types: -// (a) Matched in the URL template. -// (b) Covered by body (if body is `*`, everything except (a) fields; -// else everything under the body field) -// (c) All other fields. -// 3. URL query parameters found in the HTTP request are mapped to (c) fields. -// 4. Any body sent with an HTTP request can contain only (b) fields. -// -// The syntax of the path template is as follows: -// -// Template = "/" Segments [ Verb ] ; -// Segments = Segment { "/" Segment } ; -// Segment = "*" | "**" | LITERAL | Variable ; -// Variable = "{" FieldPath [ "=" Segments ] "}" ; -// FieldPath = IDENT { "." IDENT } ; -// Verb = ":" LITERAL ; -// -// `*` matches a single path component, `**` zero or more path components, and -// `LITERAL` a constant. A `Variable` can match an entire path as specified -// again by a template; this nested template must not contain further variables. -// If no template is given with a variable, it matches a single path component. -// The notation `{var}` is henceforth equivalent to `{var=*}`. -// -// Use CustomHttpPattern to specify any HTTP method that is not included in the -// pattern field, such as HEAD, or "*" to leave the HTTP method unspecified for -// a given URL path rule. The wild-card rule is useful for services that provide -// content to Web (HTML) clients. -message HttpRule { - - // Determines the URL pattern is matched by this rules. This pattern can be - // used with any of the {get|put|post|delete|patch} methods. A custom method - // can be defined using the 'custom' field. - oneof pattern { - // Used for listing and getting information about resources. - string get = 2; - - // Used for updating a resource. - string put = 3; - - // Used for creating a resource. - string post = 4; - - // Used for deleting a resource. - string delete = 5; - - // Used for updating a resource. - string patch = 6; - - // Custom pattern is used for defining custom verbs. - CustomHttpPattern custom = 8; - } - - // The name of the request field whose value is mapped to the HTTP body, or - // `*` for mapping all fields not captured by the path pattern to the HTTP - // body. - string body = 7; - - // Additional HTTP bindings for the selector. Nested bindings must not - // specify a selector and must not contain additional bindings. - repeated HttpRule additional_bindings = 11; -} - -// A custom pattern is used for defining custom HTTP verb. -message CustomHttpPattern { - // The name of this custom HTTP verb. - string kind = 1; - - // The path matched by this custom verb. - string path = 2; -} diff --git a/gcloud_bigtable/_generated/_operations.proto b/gcloud_bigtable/_generated/_operations.proto deleted file mode 100644 index a358d0a..0000000 --- a/gcloud_bigtable/_generated/_operations.proto +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.longrunning; - -import "google/api/annotations.proto"; -import "google/protobuf/any.proto"; -import "google/protobuf/empty.proto"; -import "google/rpc/status.proto"; - -option java_multiple_files = true; -option java_outer_classname = "OperationsProto"; -option java_package = "com.google.longrunning"; - - -// Manages long-running operations with an API service. -// -// When an API method normally takes long time to complete, it can be designed -// to return [Operation][google.longrunning.Operation] to the client, and the client can use this -// interface to receive the real response asynchronously by polling the -// operation resource, or using `google.watcher.v1.Watcher` interface to watch -// the response, or pass the operation resource to another API (such as Google -// Cloud Pub/Sub API) to receive the response. Any API service that returns -// long-running operations should implement the `Operations` interface so -// developers can have a consistent client experience. -service Operations { - // Gets the latest state of a long-running operation. Clients may use this - // method to poll the operation result at intervals as recommended by the API - // service. - rpc GetOperation(GetOperationRequest) returns (Operation) { - option (google.api.http) = { get: "/v1/{name=operations/**}" }; - } - - // Lists operations that match the specified filter in the request. If the - // server doesn't support this method, it returns - // `google.rpc.Code.UNIMPLEMENTED`. - rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { - option (google.api.http) = { get: "/v1/{name=operations}" }; - } - - // Starts asynchronous cancellation on a long-running operation. The server - // makes a best effort to cancel the operation, but success is not - // guaranteed. If the server doesn't support this method, it returns - // `google.rpc.Code.UNIMPLEMENTED`. Clients may use - // [Operations.GetOperation] or other methods to check whether the - // cancellation succeeded or the operation completed despite cancellation. - rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { post: "/v1/{name=operations/**}:cancel" body: "*" }; - } - - // Deletes a long-running operation. It indicates the client is no longer - // interested in the operation result. It does not cancel the operation. - rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { delete: "/v1/{name=operations/**}" }; - } -} - -// This resource represents a long-running operation that is the result of a -// network API call. -message Operation { - // The name of the operation resource, which is only unique within the same - // service that originally returns it. - string name = 1; - - // Some service-specific metadata associated with the operation. It typically - // contains progress information and common metadata such as create time. - // Some services may not provide such metadata. Any method that returns a - // long-running operation should document the metadata type, if any. - google.protobuf.Any metadata = 2; - - // If the value is false, it means the operation is still in progress. - // If true, the operation is completed and the `result` is available. - bool done = 3; - - oneof result { - // The error result of the operation in case of failure. - google.rpc.Status error = 4; - - // The normal response of the operation in case of success. If the original - // method returns no data on success, such as `Delete`, the response will be - // `google.protobuf.Empty`. If the original method is standard - // `Get`/`Create`/`Update`, the response should be the resource. For other - // methods, the response should have the type `XxxResponse`, where `Xxx` - // is the original method name. For example, if the original method name - // is `TakeSnapshot()`, the inferred response type will be - // `TakeSnapshotResponse`. - google.protobuf.Any response = 5; - } -} - -// The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. -message GetOperationRequest { - // The name of the operation resource. - string name = 1; -} - -// The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. -message ListOperationsRequest { - // The name of the operation collection. - string name = 4; - - // The standard List filter. - string filter = 1; - - // The standard List page size. - int32 page_size = 2; - - // The standard List page token. - string page_token = 3; -} - -// The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. -message ListOperationsResponse { - // A list of operations that match the specified filter in the request. - repeated Operation operations = 1; - - // The standard List next-page token. - string next_page_token = 2; -} - -// The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. -message CancelOperationRequest { - // The name of the operation resource to be cancelled. - string name = 1; -} - -// The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. -message DeleteOperationRequest { - // The name of the operation resource to be deleted. - string name = 1; -} diff --git a/gcloud_bigtable/_generated/_status.proto b/gcloud_bigtable/_generated/_status.proto deleted file mode 100644 index be97fa4..0000000 --- a/gcloud_bigtable/_generated/_status.proto +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.rpc; - -import "google/protobuf/any.proto"; - -option java_multiple_files = true; -option java_outer_classname = "StatusProto"; -option java_package = "com.google.rpc"; - - -// The `Status` defines a logical error model that is suitable for different -// programming environments, including REST APIs and RPC APIs. It is used by -// [gRPC](https://github.com/grpc). The error model is designed to be: -// -// - Simple to use and understand for most users. -// - Flexible enough to meet unexpected needs. -// -// # Overview -// The `Status` message contains 3 pieces of data: error code, error message, -// and error details. The error code should be an enum value of -// [google.rpc.Code][google.rpc.Code], but it may accept additional error codes if needed. The -// error message should be a developer-facing English message that helps -// developers *understand* and *resolve* the error. If a localized user-facing -// error message is needed, it can be sent in the error details or localized by -// the client. The optional error details may contain arbitrary information -// about the error. There is a predefined set of error detail types in the -// package `google.rpc` which can be used for common error conditions. -// -// # Language mapping -// The `Status` message is the logical representation of the error model, but it -// is not necessarily the actual wire format. When the `Status` message is -// exposed in different client libraries and different wire protocols, it can be -// mapped differently. For example, it will likely be mapped to some exceptions -// in Java, but more likely mapped to some error codes in C. -// -// # Other uses -// The error model and the `Status` message can be used in a variety of -// environments - either with or without APIs - to provide consistent developer -// experience across different environments. -// -// Example uses of this error model include: -// -// - Partial errors. If a service needs to return partial errors to the client, -// it may embed the `Status` in the normal response to indicate the partial -// errors. -// -// - Workflow errors. A typical workflow has multiple steps. Each step may -// have a `Status` message for error reporting purpose. -// -// - Batch operations. If a client uses batch request and batch response, the -// `Status` message should be used directly inside batch response, one for -// each error sub-response. -// -// - Asynchronous operations. If an API call embeds asynchronous operation -// results in its response, the status of those operations should be -// represented directly using the `Status` message. -// -// - Logging. If some API errors are stored in logs, the message `Status` could -// be used directly after any stripping needed for security/privacy reasons. -// -message Status { - // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. - int32 code = 1; - - // A developer-facing error message, which should be in English. Any - // user-facing error message should be localized and sent in the - // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. - string message = 2; - - // A list of messages that carry the error details. There will be a - // common set of message types for APIs to use. - repeated google.protobuf.Any details = 3; -} diff --git a/gcloud_bigtable/_generated/_timestamp.proto b/gcloud_bigtable/_generated/_timestamp.proto deleted file mode 100644 index b135854..0000000 --- a/gcloud_bigtable/_generated/_timestamp.proto +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.protobuf; - -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "TimestampProto"; -option java_package = "com.google.protobuf"; - - -// A Timestamp represents a point in time independent of any time zone -// or calendar, represented as seconds and fractions of seconds at -// nanosecond resolution in UTC Epoch time. It is encoded using the -// Proleptic Gregorian Calendar which extends the Gregorian calendar -// backwards to year one. It is encoded assuming all minutes are 60 -// seconds long, i.e. leap seconds are "smeared" so that no leap second -// table is needed for interpretation. Range is from -// 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. -// By restricting to that range, we ensure that we can convert to -// and from RFC 3339 date strings. -// See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). -// -// Example 1: Compute Timestamp from POSIX `time()`. -// -// Timestamp timestamp; -// timestamp.set_seconds(time(NULL)); -// timestamp.set_nanos(0); -// -// Example 2: Compute Timestamp from POSIX `gettimeofday()`. -// -// struct timeval tv; -// gettimeofday(&tv, NULL); -// -// Timestamp timestamp; -// timestamp.set_seconds(tv.tv_sec); -// timestamp.set_nanos(tv.tv_usec * 1000); -// -// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. -// -// FILETIME ft; -// GetSystemTimeAsFileTime(&ft); -// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; -// -// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z -// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. -// Timestamp timestamp; -// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); -// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); -// -// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. -// -// long millis = System.currentTimeMillis(); -// -// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) -// .setNanos((int) ((millis % 1000) * 1000000)).build(); -// -// Example 5: Compute Timestamp from Python `datetime.datetime`. -// -// now = datetime.datetime.utcnow() -// seconds = int(time.mktime(now.timetuple())) -// nanos = now.microsecond * 1000 -// timestamp = Timestamp(seconds=seconds, nanos=nanos) -// -message Timestamp { - // Represents seconds of UTC time since Unix epoch - // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to - // 9999-12-31T23:59:59Z inclusive. - int64 seconds = 1; - - // Non-negative fractions of a second at nanosecond resolution. Negative - // second values with fractions must still have non-negative nanos values - // that count forward in time. Must be from 0 to 999,999,999 - // inclusive. - int32 nanos = 2; -} diff --git a/gcloud_bigtable/_generated/annotations_pb2.py b/gcloud_bigtable/_generated/annotations_pb2.py deleted file mode 100644 index 905b53c..0000000 --- a/gcloud_bigtable/_generated/annotations_pb2.py +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/api/annotations.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import http_pb2 as google_dot_api_dot_http__pb2 -from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/api/annotations.proto', - package='google.api', - syntax='proto3', - serialized_pb=b'\n\x1cgoogle/api/annotations.proto\x12\ngoogle.api\x1a\x15google/api/http.proto\x1a google/protobuf/descriptor.proto:E\n\x04http\x12\x1e.google.protobuf.MethodOptions\x18\xb0\xca\xbc\" \x01(\x0b\x32\x14.google.api.HttpRuleB$\n\x0e\x63om.google.apiB\x10\x41nnotationsProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_api_dot_http__pb2.DESCRIPTOR,google_dot_protobuf_dot_descriptor__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -HTTP_FIELD_NUMBER = 72295728 -http = _descriptor.FieldDescriptor( - name='http', full_name='google.api.http', index=0, - number=72295728, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) - -DESCRIPTOR.extensions_by_name['http'] = http - -http.message_type = google_dot_api_dot_http__pb2._HTTPRULE -google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(http) - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\020AnnotationsProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/any_pb2.py b/gcloud_bigtable/_generated/any_pb2.py deleted file mode 100644 index 4712c50..0000000 --- a/gcloud_bigtable/_generated/any_pb2.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/any.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/any.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=b'\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"&\n\x03\x41ny\x12\x10\n\x08type_url\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\x42$\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01\xa0\x01\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_ANY = _descriptor.Descriptor( - name='Any', - full_name='google.protobuf.Any', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type_url', full_name='google.protobuf.Any.type_url', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.protobuf.Any.value', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=46, - serialized_end=84, -) - -DESCRIPTOR.message_types_by_name['Any'] = _ANY - -Any = _reflection.GeneratedProtocolMessageType('Any', (_message.Message,), dict( - DESCRIPTOR = _ANY, - __module__ = 'google.protobuf.any_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Any) - )) -_sym_db.RegisterMessage(Any) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\010AnyProtoP\001\240\001\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py index a01cd63..fa8b7cf 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py @@ -12,9 +12,9 @@ _sym_db = _symbol_database.Default() -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from gcloud_bigtable._generated import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py index 0b519bd..4f8cccd 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py @@ -12,7 +12,7 @@ from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as google_dot_bigtable_dot_admin_dot_cluster_dot_v1_dot_bigtable__cluster__data__pb2 -from gcloud_bigtable._generated import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py index 7a5bad8..7dab6ac 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py @@ -11,11 +11,11 @@ _sym_db = _symbol_database.Default() -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as google_dot_bigtable_dot_admin_dot_cluster_dot_v1_dot_bigtable__cluster__data__pb2 from gcloud_bigtable._generated import bigtable_cluster_service_messages_pb2 as google_dot_bigtable_dot_admin_dot_cluster_dot_v1_dot_bigtable__cluster__service__messages__pb2 -from gcloud_bigtable._generated import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -115,9 +115,9 @@ def early_adopter_create_BigtableClusterService_server(servicer, port, private_k import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 + import google.longrunning.operations_pb2 method_service_descriptions = { "CreateCluster": alpha_utilities.unary_unary_service_description( servicer.CreateCluster, @@ -127,7 +127,7 @@ def early_adopter_create_BigtableClusterService_server(servicer, port, private_k "DeleteCluster": alpha_utilities.unary_unary_service_description( servicer.DeleteCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + google.protobuf.empty_pb2.Empty.SerializeToString, ), "GetCluster": alpha_utilities.unary_unary_service_description( servicer.GetCluster, @@ -147,7 +147,7 @@ def early_adopter_create_BigtableClusterService_server(servicer, port, private_k "UndeleteCluster": alpha_utilities.unary_unary_service_description( servicer.UndeleteCluster, gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, - gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, + google.longrunning.operations_pb2.Operation.SerializeToString, ), "UpdateCluster": alpha_utilities.unary_unary_service_description( servicer.UpdateCluster, @@ -168,9 +168,9 @@ def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transf import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 + import google.longrunning.operations_pb2 method_invocation_descriptions = { "CreateCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, @@ -178,7 +178,7 @@ def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transf ), "DeleteCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, + google.protobuf.empty_pb2.Empty.FromString, ), "GetCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, @@ -194,7 +194,7 @@ def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transf ), "UndeleteCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.Operation.FromString, + google.longrunning.operations_pb2.Operation.FromString, ), "UpdateCluster": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, @@ -272,9 +272,9 @@ def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=Non import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 + import google.longrunning.operations_pb2 request_deserializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, @@ -286,11 +286,11 @@ def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=Non } response_serializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, } method_implementations = { @@ -317,9 +317,9 @@ def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transfo import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_data_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 + import google.longrunning.operations_pb2 request_serializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, @@ -331,11 +331,11 @@ def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transfo } response_deserializers = { ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.operations_pb2.Operation.FromString, + ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.FromString, ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, } cardinalities = { diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index 3b0163c..bd64308 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -11,10 +11,10 @@ _sym_db = _symbol_database.Default() -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 from gcloud_bigtable._generated import bigtable_service_messages_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -94,7 +94,7 @@ def early_adopter_create_BigtableService_server(servicer, port, private_key=None import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 @@ -108,7 +108,7 @@ def early_adopter_create_BigtableService_server(servicer, port, private_key=None "MutateRow": alpha_utilities.unary_unary_service_description( servicer.MutateRow, gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + google.protobuf.empty_pb2.Empty.SerializeToString, ), "ReadModifyWriteRow": alpha_utilities.unary_unary_service_description( servicer.ReadModifyWriteRow, @@ -133,7 +133,7 @@ def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=N import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 @@ -145,7 +145,7 @@ def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=N ), "MutateRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, + google.protobuf.empty_pb2.Empty.FromString, ), "ReadModifyWriteRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, @@ -209,7 +209,7 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 @@ -223,7 +223,7 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa } response_serializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, @@ -244,7 +244,7 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 @@ -258,7 +258,7 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No } response_deserializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.FromString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, diff --git a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py index 7c36fe9..fd47b56 100644 --- a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py @@ -11,8 +11,8 @@ _sym_db = _symbol_database.Default() -from gcloud_bigtable._generated import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from gcloud_bigtable._generated import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +from google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 DESCRIPTOR = _descriptor.FileDescriptor( diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py index 50c51a9..5a3c977 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py @@ -11,10 +11,10 @@ _sym_db = _symbol_database.Default() -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 +from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from gcloud_bigtable._generated import bigtable_table_data_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2 from gcloud_bigtable._generated import bigtable_table_service_messages_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 +from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 DESCRIPTOR = _descriptor.FileDescriptor( @@ -117,15 +117,15 @@ def early_adopter_create_BigtableTableService_server(servicer, port, private_key import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 method_service_descriptions = { "CreateColumnFamily": alpha_utilities.unary_unary_service_description( servicer.CreateColumnFamily, @@ -140,12 +140,12 @@ def early_adopter_create_BigtableTableService_server(servicer, port, private_key "DeleteColumnFamily": alpha_utilities.unary_unary_service_description( servicer.DeleteColumnFamily, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + google.protobuf.empty_pb2.Empty.SerializeToString, ), "DeleteTable": alpha_utilities.unary_unary_service_description( servicer.DeleteTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + google.protobuf.empty_pb2.Empty.SerializeToString, ), "GetTable": alpha_utilities.unary_unary_service_description( servicer.GetTable, @@ -160,7 +160,7 @@ def early_adopter_create_BigtableTableService_server(servicer, port, private_key "RenameTable": alpha_utilities.unary_unary_service_description( servicer.RenameTable, gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + google.protobuf.empty_pb2.Empty.SerializeToString, ), "UpdateColumnFamily": alpha_utilities.unary_unary_service_description( servicer.UpdateColumnFamily, @@ -177,15 +177,15 @@ def early_adopter_create_BigtableTableService_stub(host, port, metadata_transfor import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 method_invocation_descriptions = { "CreateColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, @@ -197,11 +197,11 @@ def early_adopter_create_BigtableTableService_stub(host, port, metadata_transfor ), "DeleteColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, + google.protobuf.empty_pb2.Empty.FromString, ), "DeleteTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, + google.protobuf.empty_pb2.Empty.FromString, ), "GetTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, @@ -213,7 +213,7 @@ def early_adopter_create_BigtableTableService_stub(host, port, metadata_transfor ), "RenameTable": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, + google.protobuf.empty_pb2.Empty.FromString, ), "UpdateColumnFamily": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, @@ -294,15 +294,15 @@ def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 request_deserializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, @@ -316,11 +316,11 @@ def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, response_serializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, } method_implementations = { @@ -344,15 +344,15 @@ def beta_create_BigtableTableService_stub(channel, host=None, metadata_transform import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_data_pb2 import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 + import google.protobuf.empty_pb2 request_serializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, @@ -366,11 +366,11 @@ def beta_create_BigtableTableService_stub(channel, host=None, metadata_transform response_deserializers = { ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, + ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.FromString, ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, } cardinalities = { diff --git a/gcloud_bigtable/_generated/duration_pb2.py b/gcloud_bigtable/_generated/duration_pb2.py deleted file mode 100644 index 12fd411..0000000 --- a/gcloud_bigtable/_generated/duration_pb2.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/duration.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/duration.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=b'\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\"*\n\x08\x44uration\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42)\n\x13\x63om.google.protobufB\rDurationProtoP\x01\xa0\x01\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_DURATION = _descriptor.Descriptor( - name='Duration', - full_name='google.protobuf.Duration', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='seconds', full_name='google.protobuf.Duration.seconds', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='nanos', full_name='google.protobuf.Duration.nanos', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=51, - serialized_end=93, -) - -DESCRIPTOR.message_types_by_name['Duration'] = _DURATION - -Duration = _reflection.GeneratedProtocolMessageType('Duration', (_message.Message,), dict( - DESCRIPTOR = _DURATION, - __module__ = 'google.protobuf.duration_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Duration) - )) -_sym_db.RegisterMessage(Duration) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\rDurationProtoP\001\240\001\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/empty_pb2.py b/gcloud_bigtable/_generated/empty_pb2.py deleted file mode 100644 index 5dfc0ae..0000000 --- a/gcloud_bigtable/_generated/empty_pb2.py +++ /dev/null @@ -1,62 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/empty.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/empty.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=b'\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB#\n\x13\x63om.google.protobufB\nEmptyProtoP\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_EMPTY = _descriptor.Descriptor( - name='Empty', - full_name='google.protobuf.Empty', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=48, - serialized_end=55, -) - -DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY - -Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), dict( - DESCRIPTOR = _EMPTY, - __module__ = 'google.protobuf.empty_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Empty) - )) -_sym_db.RegisterMessage(Empty) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\nEmptyProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/http_pb2.py b/gcloud_bigtable/_generated/http_pb2.py deleted file mode 100644 index a70ba4d..0000000 --- a/gcloud_bigtable/_generated/http_pb2.py +++ /dev/null @@ -1,187 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/api/http.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/api/http.proto', - package='google.api', - syntax='proto3', - serialized_pb=b'\n\x15google/api/http.proto\x12\ngoogle.api\"\xd8\x01\n\x08HttpRule\x12\r\n\x03get\x18\x02 \x01(\tH\x00\x12\r\n\x03put\x18\x03 \x01(\tH\x00\x12\x0e\n\x04post\x18\x04 \x01(\tH\x00\x12\x10\n\x06\x64\x65lete\x18\x05 \x01(\tH\x00\x12\x0f\n\x05patch\x18\x06 \x01(\tH\x00\x12/\n\x06\x63ustom\x18\x08 \x01(\x0b\x32\x1d.google.api.CustomHttpPatternH\x00\x12\x0c\n\x04\x62ody\x18\x07 \x01(\t\x12\x31\n\x13\x61\x64\x64itional_bindings\x18\x0b \x03(\x0b\x32\x14.google.api.HttpRuleB\t\n\x07pattern\"/\n\x11\x43ustomHttpPattern\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\tB\x1d\n\x0e\x63om.google.apiB\tHttpProtoP\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_HTTPRULE = _descriptor.Descriptor( - name='HttpRule', - full_name='google.api.HttpRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='get', full_name='google.api.HttpRule.get', index=0, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='put', full_name='google.api.HttpRule.put', index=1, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='post', full_name='google.api.HttpRule.post', index=2, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete', full_name='google.api.HttpRule.delete', index=3, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='patch', full_name='google.api.HttpRule.patch', index=4, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='custom', full_name='google.api.HttpRule.custom', index=5, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='body', full_name='google.api.HttpRule.body', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='additional_bindings', full_name='google.api.HttpRule.additional_bindings', index=7, - number=11, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='pattern', full_name='google.api.HttpRule.pattern', - index=0, containing_type=None, fields=[]), - ], - serialized_start=38, - serialized_end=254, -) - - -_CUSTOMHTTPPATTERN = _descriptor.Descriptor( - name='CustomHttpPattern', - full_name='google.api.CustomHttpPattern', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='kind', full_name='google.api.CustomHttpPattern.kind', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='path', full_name='google.api.CustomHttpPattern.path', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=256, - serialized_end=303, -) - -_HTTPRULE.fields_by_name['custom'].message_type = _CUSTOMHTTPPATTERN -_HTTPRULE.fields_by_name['additional_bindings'].message_type = _HTTPRULE -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['get']) -_HTTPRULE.fields_by_name['get'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['put']) -_HTTPRULE.fields_by_name['put'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['post']) -_HTTPRULE.fields_by_name['post'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['delete']) -_HTTPRULE.fields_by_name['delete'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['patch']) -_HTTPRULE.fields_by_name['patch'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['custom']) -_HTTPRULE.fields_by_name['custom'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -DESCRIPTOR.message_types_by_name['HttpRule'] = _HTTPRULE -DESCRIPTOR.message_types_by_name['CustomHttpPattern'] = _CUSTOMHTTPPATTERN - -HttpRule = _reflection.GeneratedProtocolMessageType('HttpRule', (_message.Message,), dict( - DESCRIPTOR = _HTTPRULE, - __module__ = 'google.api.http_pb2' - # @@protoc_insertion_point(class_scope:google.api.HttpRule) - )) -_sym_db.RegisterMessage(HttpRule) - -CustomHttpPattern = _reflection.GeneratedProtocolMessageType('CustomHttpPattern', (_message.Message,), dict( - DESCRIPTOR = _CUSTOMHTTPPATTERN, - __module__ = 'google.api.http_pb2' - # @@protoc_insertion_point(class_scope:google.api.CustomHttpPattern) - )) -_sym_db.RegisterMessage(CustomHttpPattern) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.apiB\tHttpProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/operations_grpc_pb2.py b/gcloud_bigtable/_generated/operations_grpc_pb2.py new file mode 100644 index 0000000..998d244 --- /dev/null +++ b/gcloud_bigtable/_generated/operations_grpc_pb2.py @@ -0,0 +1,205 @@ +import abc +from grpc.beta import implementations as beta_implementations +from grpc.early_adopter import implementations as early_adopter_implementations +from grpc.framework.alpha import utilities as alpha_utilities +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities +class EarlyAdopterOperationsServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ListOperations(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CancelOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteOperation(self, request, context): + raise NotImplementedError() +class EarlyAdopterOperationsServer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def start(self): + raise NotImplementedError() + @abc.abstractmethod + def stop(self): + raise NotImplementedError() +class EarlyAdopterOperationsStub(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request): + raise NotImplementedError() + GetOperation.async = None + @abc.abstractmethod + def ListOperations(self, request): + raise NotImplementedError() + ListOperations.async = None + @abc.abstractmethod + def CancelOperation(self, request): + raise NotImplementedError() + CancelOperation.async = None + @abc.abstractmethod + def DeleteOperation(self, request): + raise NotImplementedError() + DeleteOperation.async = None +def early_adopter_create_Operations_server(servicer, port, private_key=None, certificate_chain=None): + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + method_service_descriptions = { + "CancelOperation": alpha_utilities.unary_unary_service_description( + servicer.CancelOperation, + google.longrunning.operations_pb2.CancelOperationRequest.FromString, + google.protobuf.empty_pb2.Empty.SerializeToString, + ), + "DeleteOperation": alpha_utilities.unary_unary_service_description( + servicer.DeleteOperation, + google.longrunning.operations_pb2.DeleteOperationRequest.FromString, + google.protobuf.empty_pb2.Empty.SerializeToString, + ), + "GetOperation": alpha_utilities.unary_unary_service_description( + servicer.GetOperation, + google.longrunning.operations_pb2.GetOperationRequest.FromString, + google.longrunning.operations_pb2.Operation.SerializeToString, + ), + "ListOperations": alpha_utilities.unary_unary_service_description( + servicer.ListOperations, + google.longrunning.operations_pb2.ListOperationsRequest.FromString, + google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, + ), + } + return early_adopter_implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) +def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + method_invocation_descriptions = { + "CancelOperation": alpha_utilities.unary_unary_invocation_description( + google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, + google.protobuf.empty_pb2.Empty.FromString, + ), + "DeleteOperation": alpha_utilities.unary_unary_invocation_description( + google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, + google.protobuf.empty_pb2.Empty.FromString, + ), + "GetOperation": alpha_utilities.unary_unary_invocation_description( + google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, + google.longrunning.operations_pb2.Operation.FromString, + ), + "ListOperations": alpha_utilities.unary_unary_invocation_description( + google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, + google.longrunning.operations_pb2.ListOperationsResponse.FromString, + ), + } + return early_adopter_implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) + +class BetaOperationsServicer(object): + """""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def ListOperations(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def CancelOperation(self, request, context): + raise NotImplementedError() + @abc.abstractmethod + def DeleteOperation(self, request, context): + raise NotImplementedError() + +class BetaOperationsStub(object): + """The interface to which stubs will conform.""" + __metaclass__ = abc.ABCMeta + @abc.abstractmethod + def GetOperation(self, request, timeout): + raise NotImplementedError() + GetOperation.future = None + @abc.abstractmethod + def ListOperations(self, request, timeout): + raise NotImplementedError() + ListOperations.future = None + @abc.abstractmethod + def CancelOperation(self, request, timeout): + raise NotImplementedError() + CancelOperation.future = None + @abc.abstractmethod + def DeleteOperation(self, request, timeout): + raise NotImplementedError() + DeleteOperation.future = None + +def beta_create_Operations_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + request_deserializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.FromString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.FromString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.FromString, + } + response_serializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, + } + method_implementations = { + ('google.longrunning.Operations', 'CancelOperation'): face_utilities.unary_unary_inline(servicer.CancelOperation), + ('google.longrunning.Operations', 'DeleteOperation'): face_utilities.unary_unary_inline(servicer.DeleteOperation), + ('google.longrunning.Operations', 'GetOperation'): face_utilities.unary_unary_inline(servicer.GetOperation), + ('google.longrunning.Operations', 'ListOperations'): face_utilities.unary_unary_inline(servicer.ListOperations), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + +def beta_create_Operations_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + import google.longrunning.operations_pb2 + import google.protobuf.empty_pb2 + request_serializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, + } + response_deserializers = { + ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.FromString, + ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.FromString, + ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.FromString, + } + cardinalities = { + 'CancelOperation': cardinality.Cardinality.UNARY_UNARY, + 'DeleteOperation': cardinality.Cardinality.UNARY_UNARY, + 'GetOperation': cardinality.Cardinality.UNARY_UNARY, + 'ListOperations': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'google.longrunning.Operations', cardinalities, options=stub_options) diff --git a/gcloud_bigtable/_generated/operations_pb2.py b/gcloud_bigtable/_generated/operations_pb2.py deleted file mode 100644 index 069453b..0000000 --- a/gcloud_bigtable/_generated/operations_pb2.py +++ /dev/null @@ -1,543 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/longrunning/operations.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import any_pb2 as google_dot_protobuf_dot_any__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 -from gcloud_bigtable._generated import status_pb2 as google_dot_rpc_dot_status__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/longrunning/operations.proto', - package='google.longrunning', - syntax='proto3', - serialized_pb=b'\n#google/longrunning/operations.proto\x12\x12google.longrunning\x1a\x1cgoogle/api/annotations.proto\x1a\x19google/protobuf/any.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x17google/rpc/status.proto\"\xa8\x01\n\tOperation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x08metadata\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12#\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12(\n\x08response\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x42\x08\n\x06result\"#\n\x13GetOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\\\n\x15ListOperationsRequest\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"d\n\x16ListOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"&\n\x16\x43\x61ncelOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n\x16\x44\x65leteOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t2\x8c\x04\n\nOperations\x12x\n\x0cGetOperation\x12\'.google.longrunning.GetOperationRequest\x1a\x1d.google.longrunning.Operation\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v1/{name=operations/**}\x12\x86\x01\n\x0eListOperations\x12).google.longrunning.ListOperationsRequest\x1a*.google.longrunning.ListOperationsResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v1/{name=operations}\x12\x81\x01\n\x0f\x43\x61ncelOperation\x12*.google.longrunning.CancelOperationRequest\x1a\x16.google.protobuf.Empty\"*\x82\xd3\xe4\x93\x02$\"\x1f/v1/{name=operations/**}:cancel:\x01*\x12w\n\x0f\x44\x65leteOperation\x12*.google.longrunning.DeleteOperationRequest\x1a\x16.google.protobuf.Empty\" \x82\xd3\xe4\x93\x02\x1a*\x18/v1/{name=operations/**}B+\n\x16\x63om.google.longrunningB\x0fOperationsProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_any__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,google_dot_rpc_dot_status__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_OPERATION = _descriptor.Descriptor( - name='Operation', - full_name='google.longrunning.Operation', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.Operation.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='metadata', full_name='google.longrunning.Operation.metadata', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='done', full_name='google.longrunning.Operation.done', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='error', full_name='google.longrunning.Operation.error', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='response', full_name='google.longrunning.Operation.response', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='result', full_name='google.longrunning.Operation.result', - index=0, containing_type=None, fields=[]), - ], - serialized_start=171, - serialized_end=339, -) - - -_GETOPERATIONREQUEST = _descriptor.Descriptor( - name='GetOperationRequest', - full_name='google.longrunning.GetOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.GetOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=341, - serialized_end=376, -) - - -_LISTOPERATIONSREQUEST = _descriptor.Descriptor( - name='ListOperationsRequest', - full_name='google.longrunning.ListOperationsRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.ListOperationsRequest.name', index=0, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filter', full_name='google.longrunning.ListOperationsRequest.filter', index=1, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='page_size', full_name='google.longrunning.ListOperationsRequest.page_size', index=2, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='page_token', full_name='google.longrunning.ListOperationsRequest.page_token', index=3, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=378, - serialized_end=470, -) - - -_LISTOPERATIONSRESPONSE = _descriptor.Descriptor( - name='ListOperationsResponse', - full_name='google.longrunning.ListOperationsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='operations', full_name='google.longrunning.ListOperationsResponse.operations', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='next_page_token', full_name='google.longrunning.ListOperationsResponse.next_page_token', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=472, - serialized_end=572, -) - - -_CANCELOPERATIONREQUEST = _descriptor.Descriptor( - name='CancelOperationRequest', - full_name='google.longrunning.CancelOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.CancelOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=574, - serialized_end=612, -) - - -_DELETEOPERATIONREQUEST = _descriptor.Descriptor( - name='DeleteOperationRequest', - full_name='google.longrunning.DeleteOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.DeleteOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=614, - serialized_end=652, -) - -_OPERATION.fields_by_name['metadata'].message_type = google_dot_protobuf_dot_any__pb2._ANY -_OPERATION.fields_by_name['error'].message_type = google_dot_rpc_dot_status__pb2._STATUS -_OPERATION.fields_by_name['response'].message_type = google_dot_protobuf_dot_any__pb2._ANY -_OPERATION.oneofs_by_name['result'].fields.append( - _OPERATION.fields_by_name['error']) -_OPERATION.fields_by_name['error'].containing_oneof = _OPERATION.oneofs_by_name['result'] -_OPERATION.oneofs_by_name['result'].fields.append( - _OPERATION.fields_by_name['response']) -_OPERATION.fields_by_name['response'].containing_oneof = _OPERATION.oneofs_by_name['result'] -_LISTOPERATIONSRESPONSE.fields_by_name['operations'].message_type = _OPERATION -DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION -DESCRIPTOR.message_types_by_name['GetOperationRequest'] = _GETOPERATIONREQUEST -DESCRIPTOR.message_types_by_name['ListOperationsRequest'] = _LISTOPERATIONSREQUEST -DESCRIPTOR.message_types_by_name['ListOperationsResponse'] = _LISTOPERATIONSRESPONSE -DESCRIPTOR.message_types_by_name['CancelOperationRequest'] = _CANCELOPERATIONREQUEST -DESCRIPTOR.message_types_by_name['DeleteOperationRequest'] = _DELETEOPERATIONREQUEST - -Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), dict( - DESCRIPTOR = _OPERATION, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.Operation) - )) -_sym_db.RegisterMessage(Operation) - -GetOperationRequest = _reflection.GeneratedProtocolMessageType('GetOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _GETOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.GetOperationRequest) - )) -_sym_db.RegisterMessage(GetOperationRequest) - -ListOperationsRequest = _reflection.GeneratedProtocolMessageType('ListOperationsRequest', (_message.Message,), dict( - DESCRIPTOR = _LISTOPERATIONSREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.ListOperationsRequest) - )) -_sym_db.RegisterMessage(ListOperationsRequest) - -ListOperationsResponse = _reflection.GeneratedProtocolMessageType('ListOperationsResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTOPERATIONSRESPONSE, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.ListOperationsResponse) - )) -_sym_db.RegisterMessage(ListOperationsResponse) - -CancelOperationRequest = _reflection.GeneratedProtocolMessageType('CancelOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _CANCELOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.CancelOperationRequest) - )) -_sym_db.RegisterMessage(CancelOperationRequest) - -DeleteOperationRequest = _reflection.GeneratedProtocolMessageType('DeleteOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETEOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.DeleteOperationRequest) - )) -_sym_db.RegisterMessage(DeleteOperationRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.longrunningB\017OperationsProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterOperationsServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListOperations(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CancelOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteOperation(self, request, context): - raise NotImplementedError() -class EarlyAdopterOperationsServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterOperationsStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request): - raise NotImplementedError() - GetOperation.async = None - @abc.abstractmethod - def ListOperations(self, request): - raise NotImplementedError() - ListOperations.async = None - @abc.abstractmethod - def CancelOperation(self, request): - raise NotImplementedError() - CancelOperation.async = None - @abc.abstractmethod - def DeleteOperation(self, request): - raise NotImplementedError() - DeleteOperation.async = None -def early_adopter_create_Operations_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_service_descriptions = { - "CancelOperation": alpha_utilities.unary_unary_service_description( - servicer.CancelOperation, - gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "DeleteOperation": alpha_utilities.unary_unary_service_description( - servicer.DeleteOperation, - gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "GetOperation": alpha_utilities.unary_unary_service_description( - servicer.GetOperation, - gcloud_bigtable._generated.operations_pb2.GetOperationRequest.FromString, - gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, - ), - "ListOperations": alpha_utilities.unary_unary_service_description( - servicer.ListOperations, - gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.FromString, - gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.SerializeToString, - ), - } - return early_adopter_implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_invocation_descriptions = { - "CancelOperation": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "DeleteOperation": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "GetOperation": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.GetOperationRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.Operation.FromString, - ), - "ListOperations": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.FromString, - ), - } - return early_adopter_implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) - -class BetaOperationsServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListOperations(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CancelOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteOperation(self, request, context): - raise NotImplementedError() - -class BetaOperationsStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, timeout): - raise NotImplementedError() - GetOperation.future = None - @abc.abstractmethod - def ListOperations(self, request, timeout): - raise NotImplementedError() - ListOperations.future = None - @abc.abstractmethod - def CancelOperation(self, request, timeout): - raise NotImplementedError() - CancelOperation.future = None - @abc.abstractmethod - def DeleteOperation(self, request, timeout): - raise NotImplementedError() - DeleteOperation.future = None - -def beta_create_Operations_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - request_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.FromString, - ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.GetOperationRequest.FromString, - ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.FromString, - } - response_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.SerializeToString, - } - method_implementations = { - ('google.longrunning.Operations', 'CancelOperation'): face_utilities.unary_unary_inline(servicer.CancelOperation), - ('google.longrunning.Operations', 'DeleteOperation'): face_utilities.unary_unary_inline(servicer.DeleteOperation), - ('google.longrunning.Operations', 'GetOperation'): face_utilities.unary_unary_inline(servicer.GetOperation), - ('google.longrunning.Operations', 'ListOperations'): face_utilities.unary_unary_inline(servicer.ListOperations), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_Operations_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - request_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.GetOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.SerializeToString, - } - response_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'GetOperation'): gcloud_bigtable._generated.operations_pb2.Operation.FromString, - ('google.longrunning.Operations', 'ListOperations'): gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.FromString, - } - cardinalities = { - 'CancelOperation': cardinality.Cardinality.UNARY_UNARY, - 'DeleteOperation': cardinality.Cardinality.UNARY_UNARY, - 'GetOperation': cardinality.Cardinality.UNARY_UNARY, - 'ListOperations': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'google.longrunning.Operations', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/status_pb2.py b/gcloud_bigtable/_generated/status_pb2.py deleted file mode 100644 index 1883dcc..0000000 --- a/gcloud_bigtable/_generated/status_pb2.py +++ /dev/null @@ -1,86 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/rpc/status.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import any_pb2 as google_dot_protobuf_dot_any__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/rpc/status.proto', - package='google.rpc', - syntax='proto3', - serialized_pb=b'\n\x17google/rpc/status.proto\x12\ngoogle.rpc\x1a\x19google/protobuf/any.proto\"N\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12%\n\x07\x64\x65tails\x18\x03 \x03(\x0b\x32\x14.google.protobuf.AnyB\x1f\n\x0e\x63om.google.rpcB\x0bStatusProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_STATUS = _descriptor.Descriptor( - name='Status', - full_name='google.rpc.Status', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='google.rpc.Status.code', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='message', full_name='google.rpc.Status.message', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='details', full_name='google.rpc.Status.details', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=66, - serialized_end=144, -) - -_STATUS.fields_by_name['details'].message_type = google_dot_protobuf_dot_any__pb2._ANY -DESCRIPTOR.message_types_by_name['Status'] = _STATUS - -Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), dict( - DESCRIPTOR = _STATUS, - __module__ = 'google.rpc.status_pb2' - # @@protoc_insertion_point(class_scope:google.rpc.Status) - )) -_sym_db.RegisterMessage(Status) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\016com.google.rpcB\013StatusProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/timestamp_pb2.py b/gcloud_bigtable/_generated/timestamp_pb2.py deleted file mode 100644 index cc1ee18..0000000 --- a/gcloud_bigtable/_generated/timestamp_pb2.py +++ /dev/null @@ -1,76 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/timestamp.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/timestamp.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=b'\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\"+\n\tTimestamp\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42*\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01\xa0\x01\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_TIMESTAMP = _descriptor.Descriptor( - name='Timestamp', - full_name='google.protobuf.Timestamp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='seconds', full_name='google.protobuf.Timestamp.seconds', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='nanos', full_name='google.protobuf.Timestamp.nanos', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=52, - serialized_end=95, -) - -DESCRIPTOR.message_types_by_name['Timestamp'] = _TIMESTAMP - -Timestamp = _reflection.GeneratedProtocolMessageType('Timestamp', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMP, - __module__ = 'google.protobuf.timestamp_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Timestamp) - )) -_sym_db.RegisterMessage(Timestamp) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\023com.google.protobufB\016TimestampProtoP\001\240\001\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 3cc39a5..271e712 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -35,7 +35,7 @@ from gcloud_bigtable._generated import bigtable_cluster_service_pb2 from gcloud_bigtable._generated import bigtable_service_pb2 from gcloud_bigtable._generated import bigtable_table_service_pb2 -from gcloud_bigtable._generated import operations_pb2 +from gcloud_bigtable._generated import operations_grpc_pb2 from gcloud_bigtable._helpers import make_stub from gcloud_bigtable._non_upstream_helpers import _FactoryMixin from gcloud_bigtable._non_upstream_helpers import get_credentials @@ -62,7 +62,7 @@ DATA_API_PORT = 443 """Data API request port.""" -OPERATIONS_STUB_FACTORY = operations_pb2.beta_create_Operations_stub +OPERATIONS_STUB_FACTORY = operations_grpc_pb2.beta_create_Operations_stub ADMIN_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.admin' """Scope for interacting with the Cluster Admin and Table Admin APIs.""" diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 61a99fc..9b139ad 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -18,12 +18,13 @@ import datetime import re +from google.longrunning import operations_pb2 + from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) -from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._non_upstream_helpers import EPOCH as _EPOCH from gcloud_bigtable.table import Table @@ -95,7 +96,7 @@ def _prepare_create_request(cluster): def _pb_timestamp_to_datetime(timestamp): """Convert a Timestamp protobuf to a datetime object. - :type timestamp: :class:`._generated.timestamp_pb2.Timestamp` + :type timestamp: :class:`google.protobuf.timestamp_pb2.Timestamp` :param timestamp: A Google returned timestamp protobuf. :rtype: :class:`datetime.datetime` @@ -113,7 +114,7 @@ def _pb_timestamp_to_datetime(timestamp): def _parse_pb_any_to_native(any_val, expected_type=None): """Convert a serialized "google.protobuf.Any" value to actual type. - :type any_val: :class:`gcloud.bigtable._generated.any_pb2.Any` + :type any_val: :class:`google.protobuf.any_pb2.Any` :param any_val: A serialized protobuf value container. :type expected_type: str @@ -135,7 +136,7 @@ def _parse_pb_any_to_native(any_val, expected_type=None): def _process_operation(operation_pb): """Processes a create protobuf response. - :type operation_pb: :class:`operations_pb2.Operation` + :type operation_pb: :class:`google.longrunning.operations_pb2.Operation` :param operation_pb: The long-running operation response from a Create/Update/Undelete cluster request. @@ -213,7 +214,7 @@ def finished(self): operation_name = ('operations/' + self._cluster.name + '/operations/%d' % (self.op_id,)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) - # We expect a `._generated.operations_pb2.Operation`. + # We expect a :class:`google.longrunning.operations_pb2.Operation` operation_pb = self._cluster._client._operations_stub.GetOperation( request_pb, self._cluster._client.timeout_seconds) @@ -394,7 +395,7 @@ def create(self): create operation. """ request_pb = _prepare_create_request(self) - # We expect an `operations_pb2.Operation`. + # We expect an :class:`google.longrunning.operations_pb2.Operation` cluster_pb = self._client._cluster_stub.CreateCluster( request_pb, self._client.timeout_seconds) @@ -458,7 +459,7 @@ def delete(self): permanently deleted. """ request_pb = messages_pb2.DeleteClusterRequest(name=self.name) - # We expect a `._generated.empty_pb2.Empty` + # We expect a `google.protobuf.empty_pb2.Empty` self._client._cluster_stub.DeleteCluster( request_pb, self._client.timeout_seconds) @@ -488,7 +489,7 @@ def undelete(self): undelete operation. """ request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) - # We expect a `._generated.operations_pb2.Operation` + # We expect a :class:`google.longrunning.operations_pb2.Operation` operation_pb2 = self._client._cluster_stub.UndeleteCluster( request_pb, self._client.timeout_seconds) diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index eabdd50..6706d7e 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -17,10 +17,11 @@ import datetime +from google.protobuf import duration_pb2 + from gcloud_bigtable._generated import bigtable_table_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import duration_pb2 from gcloud_bigtable._non_upstream_helpers import _total_seconds diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 2d00b50..433b45e 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -418,7 +418,7 @@ def _commit_mutate(self, timeout_seconds=None): mutations=mutations_list, ) timeout_seconds = timeout_seconds or self.timeout_seconds - # We expect a `._generated.empty_pb2.Empty`. + # We expect a `google.protobuf.empty_pb2.Empty` self.client._data_stub.MutateRow(request_pb, timeout_seconds) def _commit_check_and_mutate(self, timeout_seconds=None): diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index c604c4a..861181e 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -181,7 +181,7 @@ def rename(self, new_table_id): new_id=new_table_id, ) client = self._cluster._client - # We expect a `._generated.empty_pb2.Empty` + # We expect a `google.protobuf.empty_pb2.Empty` client._table_stub.RenameTable(request_pb, client.timeout_seconds) self.table_id = new_table_id @@ -190,7 +190,7 @@ def delete(self): """Delete this table.""" request_pb = messages_pb2.DeleteTableRequest(name=self.name) client = self._cluster._client - # We expect a `._generated.empty_pb2.Empty` + # We expect a `google.protobuf.empty_pb2.Empty` client._table_stub.DeleteTable(request_pb, client.timeout_seconds) def list_column_families(self): diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index 4ab5964..ee437b9 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -79,9 +79,9 @@ def _callFUT(self, operation_pb): return _process_operation(operation_pb) def test_it(self): + from google.longrunning import operations_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT @@ -186,7 +186,7 @@ def test_finished_without_operation(self): def _finished_helper(self, done): import datetime - from gcloud_bigtable._generated import operations_pb2 + from google.longrunning import operations_pb2 from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable.cluster import Cluster @@ -433,9 +433,9 @@ def test_reload(self): )]) def test_create(self): + from google.longrunning import operations_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _Monkey @@ -479,9 +479,9 @@ def test_create(self): mock_process_operation.check_called(self, [(current_op,)]) def test_update(self): + from google.longrunning import operations_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _Monkey @@ -532,9 +532,9 @@ def test_update(self): mock_process_operation.check_called(self, [(current_op,)]) def test_delete(self): + from google.protobuf import empty_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock timeout_seconds = 57 @@ -565,9 +565,9 @@ def test_delete(self): )]) def test_undelete(self): + from google.longrunning import operations_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import operations_pb2 from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _Monkey @@ -674,7 +674,7 @@ def _callFUT(self, timestamp): def test_it(self): import datetime import pytz - from gcloud_bigtable._generated.timestamp_pb2 import Timestamp + from google.protobuf.timestamp_pb2 import Timestamp # Epoch is midnight on January 1, 1970 ... dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0, @@ -693,7 +693,7 @@ def _callFUT(self, any_val, expected_type=None): return _parse_pb_any_to_native(any_val, expected_type=expected_type) def test_it(self): - from gcloud_bigtable._generated import any_pb2 + from google.protobuf import any_pb2 from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT @@ -715,7 +715,7 @@ def test_it(self): self.assertEqual(result, cell) def test_unknown_type_url(self): - from gcloud_bigtable._generated import any_pb2 + from google.protobuf import any_pb2 from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT @@ -726,7 +726,7 @@ def test_unknown_type_url(self): self._callFUT(any_val) def test_disagreeing_type_url(self): - from gcloud_bigtable._generated import any_pb2 + from google.protobuf import any_pb2 from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import cluster as MUT diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py index c088552..05fe569 100644 --- a/gcloud_bigtable/test_column_family.py +++ b/gcloud_bigtable/test_column_family.py @@ -88,9 +88,9 @@ def test___ne__same_value(self): def test_to_pb(self): import datetime + from google.protobuf import duration_pb2 from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 max_age = datetime.timedelta(seconds=1) duration = duration_pb2.Duration(seconds=1) @@ -134,9 +134,9 @@ def test___ne__same_value(self): def test_to_pb(self): import datetime + from google.protobuf import duration_pb2 from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 from gcloud_bigtable.column_family import MaxAgeGCRule from gcloud_bigtable.column_family import MaxVersionsGCRule @@ -157,9 +157,9 @@ def test_to_pb(self): def test_to_pb_nested(self): import datetime + from google.protobuf import duration_pb2 from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 from gcloud_bigtable.column_family import MaxAgeGCRule from gcloud_bigtable.column_family import MaxVersionsGCRule @@ -222,9 +222,9 @@ def test___ne__same_value(self): def test_to_pb(self): import datetime + from google.protobuf import duration_pb2 from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 from gcloud_bigtable.column_family import MaxAgeGCRule from gcloud_bigtable.column_family import MaxVersionsGCRule @@ -246,9 +246,9 @@ def test_to_pb(self): def test_to_pb_nested(self): import datetime + from google.protobuf import duration_pb2 from gcloud_bigtable._generated import ( bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 from gcloud_bigtable.column_family import MaxAgeGCRule from gcloud_bigtable.column_family import MaxVersionsGCRule @@ -427,9 +427,9 @@ def test_update_with_gc_rule(self): self._update_test_helper(gc_rule=gc_rule) def test_delete(self): + from google.protobuf import empty_pb2 from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock timeout_seconds = 7 @@ -546,7 +546,7 @@ def _callFUT(self, timedelta_val): def test_it(self): import datetime - from gcloud_bigtable._generated import duration_pb2 + from google.protobuf import duration_pb2 seconds = microseconds = 1 timedelta_val = datetime.timedelta(seconds=seconds, @@ -558,7 +558,7 @@ def test_it(self): def test_with_negative_microseconds(self): import datetime - from gcloud_bigtable._generated import duration_pb2 + from google.protobuf import duration_pb2 seconds = 1 microseconds = -5 @@ -571,7 +571,7 @@ def test_with_negative_microseconds(self): def test_with_negative_seconds(self): import datetime - from gcloud_bigtable._generated import duration_pb2 + from google.protobuf import duration_pb2 seconds = -1 microseconds = 5 @@ -591,7 +591,7 @@ def _callFUT(self, duration_pb): def test_it(self): import datetime - from gcloud_bigtable._generated import duration_pb2 + from google.protobuf import duration_pb2 seconds = microseconds = 1 duration_pb = duration_pb2.Duration(seconds=seconds, diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index 0bc6a90..8f2bd2b 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -365,10 +365,10 @@ def test_delete_cells_with_string_columns(self): self.assertEqual(row._pb_mutations, [expected_pb1, expected_pb2]) def test_commit(self): + from google.protobuf import empty_pb2 from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock client = _Client() diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py index 9a430f2..6fc14be 100644 --- a/gcloud_bigtable/test_table.py +++ b/gcloud_bigtable/test_table.py @@ -141,9 +141,9 @@ def test_create_with_split_keys(self): self._create_test_helper(initial_split_keys) def test_rename(self): + from google.protobuf import empty_pb2 from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock new_table_id = 'new_table_id' @@ -182,9 +182,9 @@ def test_rename(self): )]) def test_delete(self): + from google.protobuf import empty_pb2 from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 from gcloud_bigtable._grpc_mocks import StubMock timeout_seconds = 871 diff --git a/pylintrc_default b/pylintrc_default index 912d349..6fe26e9 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -30,8 +30,6 @@ # RATIONALE: # *_pb2.py: protobuf-generated code. ignore = - annotations_pb2.py, - any_pb2.py, bigtable_cluster_data_pb2.py, bigtable_cluster_service_messages_pb2.py, bigtable_cluster_service_pb2.py, @@ -41,12 +39,6 @@ ignore = bigtable_table_data_pb2.py, bigtable_table_service_messages_pb2.py, bigtable_table_service_pb2.py, - duration_pb2.py, - empty_pb2.py, - http_pb2.py, - operations_pb2.py, - status_pb2.py, - timestamp_pb2.py, # Pickle collected data for later comparisons. # DEFAULT: persistent=yes diff --git a/scripts/make_operations_grpc.py b/scripts/make_operations_grpc.py new file mode 100644 index 0000000..d546191 --- /dev/null +++ b/scripts/make_operations_grpc.py @@ -0,0 +1,116 @@ +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Get the inserted gRPC lines for datastore pb2 file.""" + +import os +import shutil +import subprocess +import tempfile + + +ROOT_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +PROTOS_DIR = os.path.join(ROOT_DIR, 'cloud-bigtable-client', + 'bigtable-protos', 'src', 'main', 'proto') +PROTO_PATH = os.path.join(PROTOS_DIR, 'google', 'longrunning', + 'operations.proto') +GRPC_ONLY_FILE = os.path.join(ROOT_DIR, 'gcloud_bigtable', + '_generated', 'operations_grpc_pb2.py') +PROTOC_CMD = 'protoc' +GRPC_PLUGIN = 'grpc_python_plugin' + + +def get_pb2_contents_with_grpc(): + """Get pb2 lines generated by protoc with gRPC plugin. + + :rtype: list + :returns: A list of lines in the generated file. + """ + temp_dir = tempfile.mkdtemp() + generated_path = os.path.join(temp_dir, 'google', 'longrunning', + 'operations_pb2.py') + try: + subprocess.check_output([ + PROTOC_CMD, + '--proto_path', + PROTOS_DIR, + '--python_out', + temp_dir, + '--plugin', + 'protoc-gen-grpc=' + GRPC_PLUGIN, + '--grpc_out', + temp_dir, + PROTO_PATH, + ]) + with open(generated_path, 'rb') as file_obj: + return file_obj.readlines() + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def get_pb2_contents_without_grpc(): + """Get pb2 lines generated by protoc without gRPC plugin. + + :rtype: list + :returns: A list of lines in the generated file. + """ + temp_dir = tempfile.mkdtemp() + generated_path = os.path.join(temp_dir, 'google', 'longrunning', + 'operations_pb2.py') + try: + subprocess.check_output([ + PROTOC_CMD, + '--proto_path', + PROTOS_DIR, + '--python_out', + temp_dir, + PROTO_PATH, + ]) + with open(generated_path, 'rb') as file_obj: + return file_obj.readlines() + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def get_pb2_grpc_only(): + """Get pb2 lines that are only in gRPC. + + :rtype: list + :returns: A list of lines that are only in the pb2 file + generated with the gRPC plugin. + """ + grpc_contents = get_pb2_contents_with_grpc() + non_grpc_contents = get_pb2_contents_without_grpc() + + grpc_only_lines = [] + curr_non_grpc_line = 0 + for line in grpc_contents: + if line == non_grpc_contents[curr_non_grpc_line]: + curr_non_grpc_line += 1 + else: + grpc_only_lines.append(line) + + return grpc_only_lines + + +def main(): + """Write gRPC-only lines to custom module.""" + grpc_only_lines = get_pb2_grpc_only() + with open(GRPC_ONLY_FILE, 'wb') as file_obj: + file_obj.write(''.join(grpc_only_lines)) + + +if __name__ == '__main__': + main() diff --git a/scripts/rewrite_imports.py b/scripts/rewrite_imports.py index d985dac..1e5a6e0 100644 --- a/scripts/rewrite_imports.py +++ b/scripts/rewrite_imports.py @@ -23,22 +23,11 @@ IMPORT_TEMPLATE = 'import %s' IMPORT_FROM_TEMPLATE = 'from %s import ' -PROTOBUF_IMPORT_TEMPLATE = 'from google.protobuf import %s ' -REPLACE_PROTOBUF_IMPORT_TEMPLATE = 'from gcloud_bigtable._generated import %s ' REPLACEMENTS = { - 'google.api': 'gcloud_bigtable._generated', 'google.bigtable.admin.cluster.v1': 'gcloud_bigtable._generated', 'google.bigtable.admin.table.v1': 'gcloud_bigtable._generated', 'google.bigtable.v1': 'gcloud_bigtable._generated', - 'google.longrunning': 'gcloud_bigtable._generated', - 'google.rpc': 'gcloud_bigtable._generated', } -GOOGLE_PROTOBUF_CUSTOM = ( - 'any_pb2', - 'duration_pb2', - 'empty_pb2', - 'timestamp_pb2', -) def transform_old_to_new(line, old_module, new_module, @@ -107,11 +96,9 @@ def transform_old_to_new(line, old_module, new_module, def transform_line(line): """Transforms an import line in a PB2 module. - If the line is not an import of one of the packages in - ``REPLACEMENTS`` or ``GOOGLE_PROTOBUF_CUSTOM``, does nothing and returns - the original. Otherwise it replaces the package matched with our local - package or directly rewrites the custom ``google.protobuf`` import - statement. + If the line is not an import of one of the packages in ``REPLACEMENTS``, + does nothing and returns the original. Otherwise it replaces the package + matched with our local package. :type line: str :param line: The line to be transformed. @@ -124,26 +111,6 @@ def transform_line(line): if result is not None: return result - for custom_protobuf_module in GOOGLE_PROTOBUF_CUSTOM: - # We don't use the "from * import" check in transform_old_to_new - # because part of `google.protobuf` comes from the installed - # `protobuf` library. - import_from_statement = PROTOBUF_IMPORT_TEMPLATE % ( - custom_protobuf_module,) - if line.startswith(import_from_statement): - new_import_from_statement = REPLACE_PROTOBUF_IMPORT_TEMPLATE % ( - custom_protobuf_module,) - # Only replace the first instance of the import statement. - return line.replace(import_from_statement, - new_import_from_statement, 1) - - old_module = 'google.protobuf.' + custom_protobuf_module - new_module = 'gcloud_bigtable._generated.' + custom_protobuf_module - result = transform_old_to_new(line, old_module, new_module, - ignore_import_from=True) - if result is not None: - return result - # If no matches, there is nothing to transform. return line diff --git a/setup.py b/setup.py index b8c31da..6fe763e 100644 --- a/setup.py +++ b/setup.py @@ -12,9 +12,10 @@ REQUIREMENTS = [ 'httplib2 >= 0.9.1', 'oauth2client >= 1.4.6', - 'protobuf >= 3.0.0a3', + 'protobuf >= 3.0.0b2', 'pytz', 'six >= 1.6.1', + 'googleapis-common-protos', ] setup( From 42864469cafd3db632afa3243faf317adc35fbd8 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 20 Jan 2016 14:47:55 -0800 Subject: [PATCH 44/65] Merging some upstream changes. --- gcloud_bigtable/_non_upstream_helpers.py | 18 ++++++++++++++++ gcloud_bigtable/cluster.py | 27 ++++-------------------- gcloud_bigtable/column_family.py | 14 +++++++----- gcloud_bigtable/happybase/connection.py | 11 +++++----- gcloud_bigtable/row.py | 4 ++-- gcloud_bigtable/table.py | 7 +++--- 6 files changed, 43 insertions(+), 38 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 78c994a..3d38b2d 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -322,3 +322,21 @@ def _total_seconds(offset): return _total_seconds_backport(offset) else: return offset.total_seconds() + + +def _pb_timestamp_to_datetime(timestamp): + """Convert a Timestamp protobuf to a datetime object. + + :type timestamp: :class:`google.protobuf.timestamp_pb2.Timestamp` + :param timestamp: A Google returned timestamp protobuf. + + :rtype: :class:`datetime.datetime` + :returns: A UTC datetime object converted from a protobuf timestamp. + """ + return ( + EPOCH + + datetime.timedelta( + seconds=timestamp.seconds, + microseconds=(timestamp.nanos / 1000.0), + ) + ) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 9b139ad..9854363 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -15,7 +15,6 @@ """User friendly container for Google Cloud Bigtable Cluster.""" -import datetime import re from google.longrunning import operations_pb2 @@ -25,7 +24,7 @@ bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) -from gcloud_bigtable._non_upstream_helpers import EPOCH as _EPOCH +from gcloud_bigtable._non_upstream_helpers import _pb_timestamp_to_datetime from gcloud_bigtable.table import Table @@ -93,24 +92,6 @@ def _prepare_create_request(cluster): ) -def _pb_timestamp_to_datetime(timestamp): - """Convert a Timestamp protobuf to a datetime object. - - :type timestamp: :class:`google.protobuf.timestamp_pb2.Timestamp` - :param timestamp: A Google returned timestamp protobuf. - - :rtype: :class:`datetime.datetime` - :returns: A UTC datetime object converted from a protobuf timestamp. - """ - return ( - _EPOCH + - datetime.timedelta( - seconds=timestamp.seconds, - microseconds=(timestamp.nanos / 1000.0), - ) - ) - - def _parse_pb_any_to_native(any_val, expected_type=None): """Convert a serialized "google.protobuf.Any" value to actual type. @@ -214,7 +195,7 @@ def finished(self): operation_name = ('operations/' + self._cluster.name + '/operations/%d' % (self.op_id,)) request_pb = operations_pb2.GetOperationRequest(name=operation_name) - # We expect a :class:`google.longrunning.operations_pb2.Operation` + # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb = self._cluster._client._operations_stub.GetOperation( request_pb, self._cluster._client.timeout_seconds) @@ -395,7 +376,7 @@ def create(self): create operation. """ request_pb = _prepare_create_request(self) - # We expect an :class:`google.longrunning.operations_pb2.Operation` + # We expect a `google.longrunning.operations_pb2.Operation`. cluster_pb = self._client._cluster_stub.CreateCluster( request_pb, self._client.timeout_seconds) @@ -489,7 +470,7 @@ def undelete(self): undelete operation. """ request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) - # We expect a :class:`google.longrunning.operations_pb2.Operation` + # We expect a `google.longrunning.operations_pb2.Operation`. operation_pb2 = self._client._cluster_stub.UndeleteCluster( request_pb, self._client.timeout_seconds) diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py index 6706d7e..dad80d3 100644 --- a/gcloud_bigtable/column_family.py +++ b/gcloud_bigtable/column_family.py @@ -36,7 +36,7 @@ def _timedelta_to_duration_pb(timedelta_val): :type timedelta_val: :class:`datetime.timedelta` :param timedelta_val: A timedelta object. - :rtype: :class:`.duration_pb2.Duration` + :rtype: :class:`google.protobuf.duration_pb2.Duration` :returns: A duration object equivalent to the time delta. """ seconds_decimal = _total_seconds(timedelta_val) @@ -59,7 +59,7 @@ def _duration_pb_to_timedelta(duration_pb): The Python timedelta has a granularity of microseconds while the protobuf duration type has a duration of nanoseconds. - :type duration_pb: :class:`.duration_pb2.Duration` + :type duration_pb: :class:`google.protobuf.duration_pb2.Duration` :param duration_pb: A protobuf duration object. :rtype: :class:`datetime.timedelta` @@ -259,7 +259,9 @@ def create(self): column_family=column_family, ) client = self._table._cluster._client - # We expect a `.data_pb2.ColumnFamily` + # We expect a `.data_pb2.ColumnFamily`. We ignore it since the only + # data it contains are the GC rule and the column family ID already + # stored on this instance. client._table_stub.CreateColumnFamily(request_pb, client.timeout_seconds) @@ -276,7 +278,9 @@ def update(self): request_kwargs['gc_rule'] = self.gc_rule.to_pb() request_pb = data_pb2.ColumnFamily(**request_kwargs) client = self._table._cluster._client - # We expect a `.data_pb2.ColumnFamily` + # We expect a `.data_pb2.ColumnFamily`. We ignore it since the only + # data it contains are the GC rule and the column family ID already + # stored on this instance. client._table_stub.UpdateColumnFamily(request_pb, client.timeout_seconds) @@ -284,7 +288,7 @@ def delete(self): """Delete this column family.""" request_pb = messages_pb2.DeleteColumnFamilyRequest(name=self.name) client = self._table._cluster._client - # We expect a `._generated.empty_pb2.Empty` + # We expect a `google.protobuf.empty_pb2.Empty` client._table_stub.DeleteColumnFamily(request_pb, client.timeout_seconds) diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 45a15bd..2928fb6 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -26,8 +26,8 @@ from gcloud_bigtable.table import Table as _LowLevelTable -# Constants reproduced here for compatibility, though values are -# all null. +# Constants reproduced here for HappyBase compatibility, though values +# are all null. COMPAT_MODES = None THRIFT_TRANSPORTS = None THRIFT_PROTOCOLS = None @@ -149,14 +149,15 @@ class Connection(object): :param timeout: (Optional) The socket timeout in milliseconds. :type autoconnect: bool - :param autoconnect: Whether the connection should be :meth:`open`-ed - during construction. + :param autoconnect: (Optional) Whether the connection should be + :meth:`open`-ed during construction. :type table_prefix: str :param table_prefix: (Optional) Prefix used to construct table names. :type table_prefix_separator: str - :param table_prefix_separator: Separator used with ``table_prefix``. + :param table_prefix_separator: (Optional) Separator used with + ``table_prefix``. Defaults to ``_``. :type compat: :data:`NoneType ` :param compat: Unused parameter. Provided for compatibility with diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 433b45e..79c52ab 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -142,12 +142,12 @@ def _get_mutations(self, state=None): :raises: :class:`ValueError ` """ if state is None: - if self.filter is not None: + if self._filter is not None: raise ValueError('A filter is set on the current row, but no ' 'state given for the mutation') return self._pb_mutations else: - if self.filter is None: + if self._filter is None: raise ValueError('No filter was set on the current row, but a ' 'state was given for the mutation') if state: diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index 861181e..f45ddb7 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -196,9 +196,10 @@ def delete(self): def list_column_families(self): """List the column families owned by this table. - :rtype: dictionary with string as keys and - :class:`.column_family.ColumnFamily` as values - :returns: List of column families attached to this table. + :rtype: dict + :returns: Dictionary of column families attached to this table. Keys + are strings (column family names) and values are + :class:`.column_family.ColumnFamily` instances. :raises: :class:`ValueError ` if the column family name from the response does not agree with the computed name from the column family ID. From 4b6c1202bed2e5148bd669aeaae9a50a5a556d6c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 20 Jan 2016 14:56:01 -0800 Subject: [PATCH 45/65] Moving old function into _non_upstream_helpers. It's been moved into core. --- gcloud_bigtable/_non_upstream_helpers.py | 24 +++++++++++++++++++++++ gcloud_bigtable/cluster.py | 25 +----------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 3d38b2d..6f3a2f2 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -340,3 +340,27 @@ def _pb_timestamp_to_datetime(timestamp): microseconds=(timestamp.nanos / 1000.0), ) ) + + +def _get_pb_property_value(message_pb, property_name): + """Return a message field value. + + :type message_pb: :class:`google.protobuf.message.Message` + :param message_pb: The message to check for ``property_name``. + + :type property_name: str + :param property_name: The property value to check against. + + :rtype: object + :returns: The value of ``property_name`` set on ``message_pb``. + :raises: :class:`ValueError ` if the result returned + from the ``message_pb`` does not contain the ``property_name`` + value. + """ + # Make sure `property_name` is set on the response. + # NOTE: As of proto3, HasField() only works for message fields, not for + # singular (non-message) fields. + all_fields = set([field.name for field in message_pb._fields]) + if property_name not in all_fields: + raise ValueError('Message does not contain %s.' % (property_name,)) + return getattr(message_pb, property_name) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 9854363..900083c 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -24,6 +24,7 @@ bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) +from gcloud_bigtable._non_upstream_helpers import _get_pb_property_value from gcloud_bigtable._non_upstream_helpers import _pb_timestamp_to_datetime from gcloud_bigtable.table import Table @@ -47,30 +48,6 @@ } -def _get_pb_property_value(message_pb, property_name): - """Return a message field value. - - :type message_pb: :class:`google.protobuf.message.Message` - :param message_pb: The message to check for ``property_name``. - - :type property_name: str - :param property_name: The property value to check against. - - :rtype: object - :returns: The value of ``property_name`` set on ``message_pb``. - :raises: :class:`ValueError ` if the result returned - from the ``message_pb`` does not contain the ``property_name`` - value. - """ - # Make sure `property_name` is set on the response. - # NOTE: As of proto3, HasField() only works for message fields, not for - # singular (non-message) fields. - all_fields = set([field.name for field in message_pb._fields]) - if property_name not in all_fields: - raise ValueError('Message does not contain %s.' % (property_name,)) - return getattr(message_pb, property_name) - - def _prepare_create_request(cluster): """Creates a protobuf request for a CreateCluster request. From 33dff3f9c8710236b84dd1911a8dc0247d580587 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 15 Feb 2016 13:18:13 -0800 Subject: [PATCH 46/65] Updating from upstream gcloud-python. --- gcloud_bigtable/_non_upstream_helpers.py | 24 ----- gcloud_bigtable/cluster.py | 9 +- gcloud_bigtable/happybase/connection.py | 98 ++++++++++---------- gcloud_bigtable/happybase/test_connection.py | 47 ++++++---- gcloud_bigtable/row.py | 78 ++++------------ gcloud_bigtable/test_cluster.py | 50 +++++----- gcloud_bigtable/test_row.py | 27 ------ gcloud_bigtable/test_table.py | 2 +- 8 files changed, 130 insertions(+), 205 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 6f3a2f2..3d38b2d 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -340,27 +340,3 @@ def _pb_timestamp_to_datetime(timestamp): microseconds=(timestamp.nanos / 1000.0), ) ) - - -def _get_pb_property_value(message_pb, property_name): - """Return a message field value. - - :type message_pb: :class:`google.protobuf.message.Message` - :param message_pb: The message to check for ``property_name``. - - :type property_name: str - :param property_name: The property value to check against. - - :rtype: object - :returns: The value of ``property_name`` set on ``message_pb``. - :raises: :class:`ValueError ` if the result returned - from the ``message_pb`` does not contain the ``property_name`` - value. - """ - # Make sure `property_name` is set on the response. - # NOTE: As of proto3, HasField() only works for message fields, not for - # singular (non-message) fields. - all_fields = set([field.name for field in message_pb._fields]) - if property_name not in all_fields: - raise ValueError('Message does not contain %s.' % (property_name,)) - return getattr(message_pb, property_name) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py index 900083c..d5670ec 100644 --- a/gcloud_bigtable/cluster.py +++ b/gcloud_bigtable/cluster.py @@ -24,7 +24,6 @@ bigtable_cluster_service_messages_pb2 as messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as table_messages_pb2) -from gcloud_bigtable._non_upstream_helpers import _get_pb_property_value from gcloud_bigtable._non_upstream_helpers import _pb_timestamp_to_datetime from gcloud_bigtable.table import Table @@ -240,8 +239,12 @@ def table(self, table_id): return Table(table_id, self) def _update_from_pb(self, cluster_pb): - self.display_name = _get_pb_property_value(cluster_pb, 'display_name') - self.serve_nodes = _get_pb_property_value(cluster_pb, 'serve_nodes') + if not cluster_pb.display_name: # Simple field (string) + raise ValueError('Cluster protobuf does not contain display_name') + if not cluster_pb.serve_nodes: # Simple field (int32) + raise ValueError('Cluster protobuf does not contain serve_nodes') + self.display_name = cluster_pb.display_name + self.serve_nodes = cluster_pb.serve_nodes @classmethod def from_pb(cls, cluster_pb, client): diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 2928fb6..3d105e6 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -15,6 +15,8 @@ """Google Cloud Bigtable HappyBase connection module.""" +import warnings + import datetime import six @@ -37,6 +39,9 @@ DEFAULT_COMPAT = None DEFAULT_PROTOCOL = None +_LEGACY_ARGS = frozenset(('host', 'port', 'compat', 'transport', 'protocol')) +_WARN = warnings.warn + def _get_cluster(timeout=None): """Gets cluster for the default project. @@ -54,15 +59,18 @@ def _get_cluster(timeout=None): :rtype: :class:`gcloud_bigtable.cluster.Cluster` :returns: The unique cluster owned by the project inferred from the environment. - :raises: :class:`ValueError ` if any of the unused + :raises: :class:`ValueError ` if there is a failed + zone or any number of clusters other than one. """ client_kwargs = {'admin': True} if timeout is not None: client_kwargs['timeout_seconds'] = timeout / 1000.0 client = Client(**client_kwargs) - client.start() - clusters, failed_zones = client.list_clusters() - client.stop() + try: + client.start() + clusters, failed_zones = client.list_clusters() + finally: + client.stop() if len(failed_zones) != 0: raise ValueError('Determining cluster via ListClusters encountered ' @@ -137,13 +145,10 @@ class Connection(object): :class:`Credentials ` stored on the client. - :type host: :data:`NoneType ` - :param host: Unused parameter. Provided for compatibility with HappyBase, - but irrelevant for Cloud Bigtable since it has a fixed host. - - :type port: :data:`NoneType ` - :param port: Unused parameter. Provided for compatibility with HappyBase, - but irrelevant for Cloud Bigtable since it has a fixed host. + The arguments ``host``, ``port``, ``compat``, ``transport`` and + ``protocol`` are allowed (as keyword arguments) for compatibility with + HappyBase. However, they will not be used in anyway, and will cause a + warning if passed. :type timeout: int :param timeout: (Optional) The socket timeout in milliseconds. @@ -159,56 +164,28 @@ class Connection(object): :param table_prefix_separator: (Optional) Separator used with ``table_prefix``. Defaults to ``_``. - :type compat: :data:`NoneType ` - :param compat: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since there - is only one version. - - :type transport: :data:`NoneType ` - :param transport: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since the - transport is fixed. - - :type protocol: :data:`NoneType ` - :param protocol: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since the - protocol is fixed. - :type cluster: :class:`gcloud_bigtable.cluster.Cluster` :param cluster: (Optional) A Cloud Bigtable cluster. The instance also owns a client for making gRPC requests to the Cloud Bigtable API. If not passed in, defaults to creating client - with ``admin=True`` and using the ``timeout`` for the - ``timeout_seconds``. The credentials for the client + with ``admin=True`` and using the ``timeout`` here for the + ``timeout_seconds`` argument to the :class:`.Client`` + constructor. The credentials for the client will be the implicit ones loaded from the environment. Then that client is used to retrieve all the clusters owned by the client's project. + :type kwargs: dict + :param kwargs: Remaining keyword arguments. Provided for HappyBase + compatibility. + :raises: :class:`ValueError ` if any of the unused parameters are specified with a value other than the defaults. - :class:`TypeError ` if ``table_prefix`` or - ``table_prefix_separator`` are provided but not strings. """ - def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, timeout=None, - autoconnect=True, table_prefix=None, - table_prefix_separator='_', compat=DEFAULT_COMPAT, - transport=DEFAULT_TRANSPORT, protocol=DEFAULT_PROTOCOL, - cluster=None): - if host is not DEFAULT_HOST: - raise ValueError('Host cannot be set for gcloud HappyBase module') - if port is not DEFAULT_PORT: - raise ValueError('Port cannot be set for gcloud HappyBase module') - if compat is not DEFAULT_COMPAT: - raise ValueError('Compat cannot be set for gcloud ' - 'HappyBase module') - if transport is not DEFAULT_TRANSPORT: - raise ValueError('Transport cannot be set for gcloud ' - 'HappyBase module') - if protocol is not DEFAULT_PROTOCOL: - raise ValueError('Protocol cannot be set for gcloud ' - 'HappyBase module') - + def __init__(self, timeout=None, autoconnect=True, table_prefix=None, + table_prefix_separator='_', cluster=None, **kwargs): + self._handle_legacy_args(kwargs) if table_prefix is not None: if not isinstance(table_prefix, six.string_types): raise TypeError('table_prefix must be a string', 'received', @@ -235,6 +212,29 @@ def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, timeout=None, self._initialized = True + @staticmethod + def _handle_legacy_args(arguments_dict): + """Check legacy HappyBase arguments and warn if set. + + :type arguments_dict: dict + :param arguments_dict: Unused keyword arguments. + + :raises: :class:`TypeError ` if a keyword other + than ``host``, ``port``, ``compat``, ``transport`` or + ``protocol`` is used. + """ + common_args = _LEGACY_ARGS.intersection(six.iterkeys(arguments_dict)) + if common_args: + all_args = ', '.join(common_args) + message = ('The HappyBase legacy arguments %s were used. These ' + 'arguments are unused by gcloud.' % (all_args,)) + _WARN(message) + for arg_name in common_args: + arguments_dict.pop(arg_name) + if arguments_dict: + unexpected_names = arguments_dict.keys() + raise TypeError('Received unexpected arguments', unexpected_names) + def open(self): """Open the underlying transport to Cloud Bigtable. diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index ce5cf67..78e7cf5 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -201,6 +201,33 @@ def test_constructor_explicit(self): table_prefix_separator) self.assertEqual(connection._cluster, cluster_copy) + def test_constructor_with_unknown_argument(self): + cluster = _Cluster() + with self.assertRaises(TypeError): + self._makeOne(cluster=cluster, unknown='foo') + + def test_constructor_with_legacy_args(self): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import connection as MUT + + mock_warn = _MockCalled() + cluster = _Cluster() + with _Monkey(MUT, _WARN=mock_warn): + self._makeOne(cluster=cluster, host=object(), + port=object(), compat=object(), + transport=object(), protocol=object()) + + self.assertEqual(mock_warn.called_kwargs, [{}]) + self.assertEqual(len(mock_warn.called_args), 1) + self.assertEqual(len(mock_warn.called_args[0]), 1) + warning_msg = mock_warn.called_args[0][0] + self.assertIn('host', warning_msg) + self.assertIn('port', warning_msg) + self.assertIn('compat', warning_msg) + self.assertIn('transport', warning_msg) + self.assertIn('protocol', warning_msg) + def test_constructor_non_string_prefix(self): table_prefix = object() @@ -215,26 +242,6 @@ def test_constructor_non_string_prefix_separator(self): self._makeOne(autoconnect=False, table_prefix_separator=table_prefix_separator) - def test_constructor_with_host(self): - with self.assertRaises(ValueError): - self._makeOne(host=object()) - - def test_constructor_with_port(self): - with self.assertRaises(ValueError): - self._makeOne(port=object()) - - def test_constructor_with_compat(self): - with self.assertRaises(ValueError): - self._makeOne(compat=object()) - - def test_constructor_with_transport(self): - with self.assertRaises(ValueError): - self._makeOne(transport=object()) - - def test_constructor_with_protocol(self): - with self.assertRaises(ValueError): - self._makeOne(protocol=object()) - def test_constructor_with_timeout_and_cluster(self): with self.assertRaises(ValueError): self._makeOne(cluster=object(), timeout=object()) diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 79c52ab..431a219 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -80,51 +80,6 @@ def __init__(self, row_key, table, filter_=None): self._true_pb_mutations = [] self._false_pb_mutations = [] - @property - def table(self): - """Getter for row's table. - - :rtype: :class:`Table ` - :returns: The table stored on the row. - """ - return self._table - - @property - def row_key(self): - """Getter for row's key. - - :rtype: bytes - :returns: The key for the row. - """ - return self._row_key - - @property - def filter(self): - """Getter for row's filter. - - :rtype: :class:`RowFilter` or :data:`NoneType ` - :returns: The filter for the row. - """ - return self._filter - - @property - def client(self): - """Getter for row's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this row. - """ - return self.table._cluster._client - - @property - def timeout_seconds(self): - """Getter for row's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default. - """ - return self.table._cluster._client.timeout_seconds - def _get_mutations(self, state=None): """Gets the list of mutations for a given state. @@ -201,7 +156,9 @@ def set_cell(self, column_family_id, column, value, timestamp=None, # Use -1 for current Bigtable server time. timestamp_micros = -1 else: - timestamp_micros = _timestamp_to_microseconds(timestamp) + timestamp_micros = _microseconds_from_datetime(timestamp) + # Truncate to millisecond granularity. + timestamp_micros -= (timestamp_micros % 1000) mutation_val = data_pb2.Mutation.SetCell( family_name=column_family_id, @@ -413,13 +370,14 @@ def _commit_mutate(self, timeout_seconds=None): raise ValueError('%d total mutations exceed the maximum allowable ' '%d.' % (num_mutations, _MAX_MUTATIONS)) request_pb = messages_pb2.MutateRowRequest( - table_name=self.table.name, - row_key=self.row_key, + table_name=self._table.name, + row_key=self._row_key, mutations=mutations_list, ) timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `google.protobuf.empty_pb2.Empty` - self.client._data_stub.MutateRow(request_pb, timeout_seconds) + client = self._table._cluster._client + client._data_stub.MutateRow(request_pb, timeout_seconds) def _commit_check_and_mutate(self, timeout_seconds=None): """Makes a ``CheckAndMutateRow`` API request. @@ -451,21 +409,22 @@ def _commit_check_and_mutate(self, timeout_seconds=None): _MAX_MUTATIONS, num_true_mutations, num_false_mutations)) request_pb = messages_pb2.CheckAndMutateRowRequest( - table_name=self.table.name, - row_key=self.row_key, - predicate_filter=self.filter.to_pb(), + table_name=self._table.name, + row_key=self._row_key, + predicate_filter=self._filter.to_pb(), true_mutations=true_mutations, false_mutations=false_mutations, ) timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.messages_pb2.CheckAndMutateRowResponse` - resp = self.client._data_stub.CheckAndMutateRow( + client = self._table._cluster._client + resp = client._data_stub.CheckAndMutateRow( request_pb, timeout_seconds) return resp.predicate_matched def clear_mutations(self): """Removes all currently accumulated mutations on the current row.""" - if self.filter is None: + if self._filter is None: self._pb_mutations[:] = [] else: self._true_pb_mutations[:] = [] @@ -499,7 +458,7 @@ def commit(self, timeout_seconds=None): :raises: :class:`ValueError ` if the number of mutations exceeds the ``_MAX_MUTATIONS``. """ - if self.filter is None: + if self._filter is None: result = self._commit_mutate(timeout_seconds=timeout_seconds) else: result = self._commit_check_and_mutate( @@ -560,19 +519,20 @@ def commit_modifications(self, timeout_seconds=None): if len(self._rule_pb_list) == 0: return {} request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=self.table.name, - row_key=self.row_key, + table_name=self._table.name, + row_key=self._row_key, rules=self._rule_pb_list, ) timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.data_pb2.Row` - row_response = self.client._data_stub.ReadModifyWriteRow( + client = self._table._cluster._client + row_response = client._data_stub.ReadModifyWriteRow( request_pb, timeout_seconds) # Reset modifications after commit-ing request. self.clear_modification_rules() - # NOTE: We expect row_response.key == self.row_key but don't check. + # NOTE: We expect row_response.key == self._row_key but don't check. return _parse_rmw_row_response(row_response) diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py index ee437b9..aef4fa5 100644 --- a/gcloud_bigtable/test_cluster.py +++ b/gcloud_bigtable/test_cluster.py @@ -21,28 +21,6 @@ CLUSTER_ID = 'cluster-id' -class Test__get_pb_property_value(unittest2.TestCase): - - def _callFUT(self, message_pb, property_name): - from gcloud_bigtable.cluster import _get_pb_property_value - return _get_pb_property_value(message_pb, property_name) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - cluster_pb = data_pb2.Cluster(serve_nodes=serve_nodes) - result = self._callFUT(cluster_pb, 'serve_nodes') - self.assertEqual(result, serve_nodes) - - def test_with_value_unset_on_pb(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - cluster_pb = data_pb2.Cluster() - with self.assertRaises(ValueError): - self._callFUT(cluster_pb, 'serve_nodes') - - class Test__prepare_create_request(unittest2.TestCase): def _callFUT(self, cluster): @@ -311,6 +289,34 @@ def test_table_factory(self): self.assertEqual(table.table_id, table_id) self.assertEqual(table._cluster, cluster) + def test__update_from_pb_no_display_name(self): + from gcloud_bigtable._generated import ( + bigtable_cluster_data_pb2 as data_pb2) + from gcloud_bigtable.cluster import _DEFAULT_SERVE_NODES + + cluster_pb = data_pb2.Cluster(serve_nodes=331) + cluster = self._makeOne(None, None, None) + self.assertEqual(cluster.display_name, None) + self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + with self.assertRaises(ValueError): + cluster._update_from_pb(cluster_pb) + self.assertEqual(cluster.display_name, None) + self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + + def test__update_from_pb_no_serve_nodes(self): + from gcloud_bigtable._generated import ( + bigtable_cluster_data_pb2 as data_pb2) + from gcloud_bigtable.cluster import _DEFAULT_SERVE_NODES + + cluster_pb = data_pb2.Cluster(display_name='name') + cluster = self._makeOne(None, None, None) + self.assertEqual(cluster.display_name, None) + self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + with self.assertRaises(ValueError): + cluster._update_from_pb(cluster_pb) + self.assertEqual(cluster.display_name, None) + self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) + def test_from_pb_success(self): from gcloud_bigtable._generated import ( bigtable_cluster_data_pb2 as data_pb2) diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index 8f2bd2b..4c7b965 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -72,33 +72,6 @@ def test_constructor_with_non_bytes(self): with self.assertRaises(TypeError): self._constructor_helper(row_key) - def test_table_getter(self): - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertTrue(row.table is table) - - def test_row_key_getter(self): - row = self._makeOne(ROW_KEY, object()) - self.assertEqual(row.row_key, ROW_KEY) - - def test_filter_getter(self): - filter_ = object() - row = self._makeOne(ROW_KEY, object(), filter_=filter_) - self.assertTrue(row.filter is filter_) - - def test_client_getter(self): - client = object() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertTrue(row.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 889 - client = _Client(timeout_seconds=timeout_seconds) - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row.timeout_seconds, timeout_seconds) - def _get_mutations_helper(self, filter_=None, state=None): row = self._makeOne(ROW_KEY, None, filter_=filter_) # Mock the mutations with unique objects so we can compare. diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py index 6fc14be..e63aac0 100644 --- a/gcloud_bigtable/test_table.py +++ b/gcloud_bigtable/test_table.py @@ -64,7 +64,7 @@ def test_row_factory(self): filter_ = object() row = table.row(row_key, filter_=filter_) self.assertTrue(isinstance(row, Row)) - self.assertEqual(row.row_key, row_key) + self.assertEqual(row._row_key, row_key) self.assertEqual(row._table, table) self.assertEqual(row._filter, filter_) From 531e3e8edf4fc52933454994ac6187dcdc3d3175 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 15 Feb 2016 13:28:52 -0800 Subject: [PATCH 47/65] Removing timeout_seconds stuff from row.py. --- gcloud_bigtable/row.py | 44 ++++++++++--------------------------- gcloud_bigtable/test_row.py | 19 ++++++++-------- 2 files changed, 21 insertions(+), 42 deletions(-) diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 431a219..30a44d2 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -349,20 +349,16 @@ def delete_cells(self, column_family_id, columns, time_range=None, # processed without error. mutations_list.extend(to_append) - def _commit_mutate(self, timeout_seconds=None): + def _commit_mutate(self): """Makes a ``MutateRow`` API request. Assumes no filter is set on the :class:`Row` and is meant to be called by :meth:`commit`. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - :raises: :class:`ValueError ` if the number of mutations exceeds the ``_MAX_MUTATIONS``. """ - mutations_list = self._get_mutations(None) + mutations_list = self._get_mutations() num_mutations = len(mutations_list) if num_mutations == 0: return @@ -374,29 +370,24 @@ def _commit_mutate(self, timeout_seconds=None): row_key=self._row_key, mutations=mutations_list, ) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `google.protobuf.empty_pb2.Empty` client = self._table._cluster._client - client._data_stub.MutateRow(request_pb, timeout_seconds) + client._data_stub.MutateRow(request_pb, client.timeout_seconds) - def _commit_check_and_mutate(self, timeout_seconds=None): + def _commit_check_and_mutate(self): """Makes a ``CheckAndMutateRow`` API request. Assumes a filter is set on the :class:`Row` and is meant to be called by :meth:`commit`. - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - :rtype: bool :returns: Flag indicating if the filter was matched (which also indicates which set of mutations were applied by the server). :raises: :class:`ValueError ` if the number of mutations exceeds the ``_MAX_MUTATIONS``. """ - true_mutations = self._get_mutations(True) - false_mutations = self._get_mutations(False) + true_mutations = self._get_mutations(state=True) + false_mutations = self._get_mutations(state=False) num_true_mutations = len(true_mutations) num_false_mutations = len(false_mutations) if num_true_mutations == 0 and num_false_mutations == 0: @@ -415,11 +406,10 @@ def _commit_check_and_mutate(self, timeout_seconds=None): true_mutations=true_mutations, false_mutations=false_mutations, ) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.messages_pb2.CheckAndMutateRowResponse` client = self._table._cluster._client resp = client._data_stub.CheckAndMutateRow( - request_pb, timeout_seconds) + request_pb, client.timeout_seconds) return resp.predicate_matched def clear_mutations(self): @@ -430,7 +420,7 @@ def clear_mutations(self): self._true_pb_mutations[:] = [] self._false_pb_mutations[:] = [] - def commit(self, timeout_seconds=None): + def commit(self): """Makes a ``MutateRow`` or ``CheckAndMutateRow`` API request. If no mutations have been created in the row, no request is made. @@ -447,10 +437,6 @@ def commit(self, timeout_seconds=None): any cells in the :class:`Row` or not. (Each method which adds a mutation has a ``state`` parameter for this purpose.) - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - :rtype: :class:`bool` or :data:`NoneType ` :returns: :data:`None` if there is no filter, otherwise a flag indicating if the filter was matched (which also @@ -459,10 +445,9 @@ def commit(self, timeout_seconds=None): mutations exceeds the ``_MAX_MUTATIONS``. """ if self._filter is None: - result = self._commit_mutate(timeout_seconds=timeout_seconds) + result = self._commit_mutate() else: - result = self._commit_check_and_mutate( - timeout_seconds=timeout_seconds) + result = self._commit_check_and_mutate() # Reset mutations after commit-ing request. self.clear_mutations() @@ -473,7 +458,7 @@ def clear_modification_rules(self): """Removes all currently accumulated modifications on current row.""" self._rule_pb_list[:] = [] - def commit_modifications(self, timeout_seconds=None): + def commit_modifications(self): """Makes a ``ReadModifyWriteRow`` API request. This commits modifications made by :meth:`append_cell_value` and @@ -486,10 +471,6 @@ def commit_modifications(self, timeout_seconds=None): or the highest timestamp of a cell in that column (if it exceeds the server time). - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - :rtype: dict :returns: The new contents of all modified cells. Returned as a dictionary of column families, each of which holds a @@ -523,11 +504,10 @@ def commit_modifications(self, timeout_seconds=None): row_key=self._row_key, rules=self._rule_pb_list, ) - timeout_seconds = timeout_seconds or self.timeout_seconds # We expect a `.data_pb2.Row` client = self._table._cluster._client row_response = client._data_stub.ReadModifyWriteRow( - request_pb, timeout_seconds) + request_pb, client.timeout_seconds) # Reset modifications after commit-ing request. self.clear_modification_rules() diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index 4c7b965..0b8ea3a 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -344,7 +344,8 @@ def test_commit(self): bigtable_service_messages_pb2 as messages_pb2) from gcloud_bigtable._grpc_mocks import StubMock - client = _Client() + timeout_seconds = 711 + client = _Client(timeout_seconds=timeout_seconds) table = _Table(TABLE_NAME, client=client) row = self._makeOne(ROW_KEY, table) @@ -374,9 +375,8 @@ def test_commit(self): expected_result = None # commit() has no return value when no filter. # Perform the method and check the result. - timeout_seconds = 711 row.set_cell(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit(timeout_seconds=timeout_seconds) + result = row.commit() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'MutateRow', @@ -423,7 +423,8 @@ def test_commit_with_filter(self): from gcloud_bigtable._grpc_mocks import StubMock from gcloud_bigtable.row import RowSampleFilter - client = _Client() + timeout_seconds = 262 + client = _Client(timeout_seconds=timeout_seconds) table = _Table(TABLE_NAME, client=client) row_filter = RowSampleFilter(0.33) row = self._makeOne(ROW_KEY, table, filter_=row_filter) @@ -458,9 +459,8 @@ def test_commit_with_filter(self): expected_result = predicate_matched # Perform the method and check the result. - timeout_seconds = 262 row.set_cell(COLUMN_FAMILY_ID, COLUMN, value, state=True) - result = row.commit(timeout_seconds=timeout_seconds) + result = row.commit() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( 'CheckAndMutateRow', @@ -512,7 +512,8 @@ def test_commit_modifications(self): from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import row as MUT - client = _Client() + timeout_seconds = 87 + client = _Client(timeout_seconds=timeout_seconds) table = _Table(TABLE_NAME, client=client) row = self._makeOne(ROW_KEY, table) @@ -542,11 +543,9 @@ def test_commit_modifications(self): mock_parse_rmw_row_response = _MockCalled(expected_result) # Perform the method and check the result. - timeout_seconds = 87 - with _Monkey(MUT, _parse_rmw_row_response=mock_parse_rmw_row_response): row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit_modifications(timeout_seconds=timeout_seconds) + result = row.commit_modifications() self.assertEqual(result, expected_result) self.assertEqual(stub.method_calls, [( From 6aa12da00a2fc86968d9f2b877bc352e5106af02 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 15 Feb 2016 14:57:57 -0800 Subject: [PATCH 48/65] Merging in more upstream changes. --- gcloud_bigtable/happybase/test_connection.py | 14 +++++++++----- gcloud_bigtable/row.py | 11 ++++++----- gcloud_bigtable/row_data.py | 7 ++++++- gcloud_bigtable/test_row.py | 3 +-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index 78e7cf5..34475bc 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -165,6 +165,8 @@ def test_constructor_defaults(self): def test_constructor_no_autoconnect(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) + self.assertEqual(cluster._client.start_calls, 0) + self.assertEqual(cluster._client.stop_calls, 0) self.assertEqual(connection.table_prefix, None) self.assertEqual(connection.table_prefix_separator, '_') @@ -186,13 +188,14 @@ def test_constructor_missing_cluster(self): mock_get_cluster.check_called(self, [()], [{'timeout': timeout}]) def test_constructor_explicit(self): + autoconnect = False table_prefix = 'table-prefix' table_prefix_separator = 'sep' cluster_copy = _Cluster() cluster = _Cluster(copies=[cluster_copy]) connection = self._makeOne( - autoconnect=False, + autoconnect=autoconnect, table_prefix=table_prefix, table_prefix_separator=table_prefix_separator, cluster=cluster) @@ -228,6 +231,11 @@ def test_constructor_with_legacy_args(self): self.assertIn('transport', warning_msg) self.assertIn('protocol', warning_msg) + def test_constructor_with_timeout_and_cluster(self): + cluster = _Cluster() + with self.assertRaises(ValueError): + self._makeOne(cluster=cluster, timeout=object()) + def test_constructor_non_string_prefix(self): table_prefix = object() @@ -242,10 +250,6 @@ def test_constructor_non_string_prefix_separator(self): self._makeOne(autoconnect=False, table_prefix_separator=table_prefix_separator) - def test_constructor_with_timeout_and_cluster(self): - with self.assertRaises(ValueError): - self._makeOne(cluster=object(), timeout=object()) - def test_open(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py index 30a44d2..3755b89 100644 --- a/gcloud_bigtable/row.py +++ b/gcloud_bigtable/row.py @@ -30,6 +30,7 @@ # pylint: disable=invalid-name # Alias for easy upstream diffs. _microseconds_from_datetime = _timestamp_to_microseconds +_datetime_from_microseconds = _microseconds_to_timestamp # pylint: enable=invalid-name _MAX_MUTATIONS = 100000 _PACK_I64 = struct.Struct('>q').pack @@ -415,10 +416,10 @@ def _commit_check_and_mutate(self): def clear_mutations(self): """Removes all currently accumulated mutations on the current row.""" if self._filter is None: - self._pb_mutations[:] = [] + del self._pb_mutations[:] else: - self._true_pb_mutations[:] = [] - self._false_pb_mutations[:] = [] + del self._true_pb_mutations[:] + del self._false_pb_mutations[:] def commit(self): """Makes a ``MutateRow`` or ``CheckAndMutateRow`` API request. @@ -456,7 +457,7 @@ def commit(self): def clear_modification_rules(self): """Removes all currently accumulated modifications on current row.""" - self._rule_pb_list[:] = [] + del self._rule_pb_list[:] def commit_modifications(self): """Makes a ``ReadModifyWriteRow`` API request. @@ -1331,7 +1332,7 @@ def _parse_family_pb(family_pb): for cell in column.cells: val_pair = ( cell.value, - _microseconds_to_timestamp(cell.timestamp_micros), + _datetime_from_microseconds(cell.timestamp_micros), ) cells.append(val_pair) diff --git a/gcloud_bigtable/row_data.py b/gcloud_bigtable/row_data.py index b8cab89..e0ec054 100644 --- a/gcloud_bigtable/row_data.py +++ b/gcloud_bigtable/row_data.py @@ -21,6 +21,11 @@ from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable._non_upstream_helpers import _to_bytes +# pylint: disable=invalid-name +# Alias for easy upstream diffs. +_datetime_from_microseconds = _microseconds_to_timestamp +# pylint: enable=invalid-name + class Cell(object): """Representation of a Google Cloud Bigtable Cell. @@ -50,7 +55,7 @@ def from_pb(cls, cell_pb): :rtype: :class:`Cell` :returns: The cell corresponding to the protobuf. """ - timestamp = _microseconds_to_timestamp(cell_pb.timestamp_micros) + timestamp = _datetime_from_microseconds(cell_pb.timestamp_micros) if cell_pb.labels: return cls(cell_pb.value, timestamp, labels=cell_pb.labels) else: diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py index 0b8ea3a..6968744 100644 --- a/gcloud_bigtable/test_row.py +++ b/gcloud_bigtable/test_row.py @@ -1730,7 +1730,6 @@ def __init__(self, client=None): class _Table(object): - def __init__(self, name, client=None, timeout_seconds=None): + def __init__(self, name, client=None): self.name = name self._cluster = _Cluster(client) - self.timeout_seconds = timeout_seconds From c04dbeecdc84182b40425567003cb25fd4fc5f8a Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 15 Feb 2016 15:06:09 -0800 Subject: [PATCH 49/65] Updating generated modules (some new service definitions). --- .../_generated/_bigtable_cluster_data.proto | 9 +- .../_generated/_bigtable_data.proto | 15 ++ .../_generated/_bigtable_service.proto | 7 + .../_bigtable_service_messages.proto | 37 ++++ .../_generated/bigtable_cluster_data_pb2.py | 29 ++- .../_generated/bigtable_data_pb2.py | 101 ++++++--- .../bigtable_service_messages_pb2.py | 192 +++++++++++++++--- .../_generated/bigtable_service_pb2.py | 39 +++- 8 files changed, 357 insertions(+), 72 deletions(-) diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_data.proto b/gcloud_bigtable/_generated/_bigtable_cluster_data.proto index cbebe18..c0f8a93 100644 --- a/gcloud_bigtable/_generated/_bigtable_cluster_data.proto +++ b/gcloud_bigtable/_generated/_bigtable_cluster_data.proto @@ -62,11 +62,6 @@ message Cluster { // projects//zones//clusters/[a-z][-a-z0-9]* string name = 1; - // If this cluster has been deleted, the time at which its backup will - // be irrevocably destroyed. Omitted otherwise. - // This cannot be set directly, only through DeleteCluster. - google.protobuf.Timestamp delete_time = 2; - // The operation currently running on the cluster, if any. // This cannot be set directly, only through CreateCluster, UpdateCluster, // or UndeleteCluster. Calls to these methods will be rejected if @@ -91,4 +86,8 @@ enum StorageType { // Data will be stored in SSD, providing low and consistent latencies. STORAGE_SSD = 1; + + // Data will be stored in HDD, providing high and less predictable + // latencies. + STORAGE_HDD = 2; } diff --git a/gcloud_bigtable/_generated/_bigtable_data.proto b/gcloud_bigtable/_generated/_bigtable_data.proto index 86234d2..290eb91 100644 --- a/gcloud_bigtable/_generated/_bigtable_data.proto +++ b/gcloud_bigtable/_generated/_bigtable_data.proto @@ -89,6 +89,15 @@ message RowRange { bytes end_key = 3; } +// Specifies a non-contiguous set of rows. +message RowSet { + // Single rows included in the set. + repeated bytes row_keys = 1; + + // Contiguous row ranges included in the set. + repeated RowRange row_ranges = 2; +} + // Specifies a contiguous range of columns within a single column family. // The range spans from : to // :, where both bounds can be either inclusive or @@ -374,15 +383,21 @@ message RowFilter { ValueRange value_range_filter = 15; // Skips the first N cells of each row, matching all subsequent cells. + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. int32 cells_per_row_offset_filter = 10; // Matches only the first N cells of each row. + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. int32 cells_per_row_limit_filter = 11; // Matches only the most recent N cells within each column. For example, // if N=2, this filter would match column "foo:bar" at timestamps 10 and 9, // skip all earlier cells in "foo:bar", and then begin matching again in // column "foo:bar2". + // If duplicate cells are present, as is possible when using an Interleave, + // each copy of the cell is counted separately. int32 cells_per_column_limit_filter = 12; // Replaces each cell's value with the empty string. diff --git a/gcloud_bigtable/_generated/_bigtable_service.proto b/gcloud_bigtable/_generated/_bigtable_service.proto index 7f9109f..f1a83d3 100644 --- a/gcloud_bigtable/_generated/_bigtable_service.proto +++ b/gcloud_bigtable/_generated/_bigtable_service.proto @@ -51,6 +51,13 @@ service BigtableService { option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate" body: "*" }; } + // Mutates multiple rows in a batch. Each individual row is mutated + // atomically as in MutateRow, but the entire batch is not executed + // atomically. + rpc MutateRows(MutateRowsRequest) returns (MutateRowsResponse) { + option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}:mutateRows" body: "*" }; + } + // Mutates a row atomically based on the output of a predicate Reader filter. rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) { option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate" body: "*" }; diff --git a/gcloud_bigtable/_generated/_bigtable_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_service_messages.proto index dacc0fc..1479fb6 100644 --- a/gcloud_bigtable/_generated/_bigtable_service_messages.proto +++ b/gcloud_bigtable/_generated/_bigtable_service_messages.proto @@ -17,6 +17,7 @@ syntax = "proto3"; package google.bigtable.v1; import "google/bigtable/v1/bigtable_data.proto"; +import "google/rpc/status.proto"; option java_multiple_files = true; option java_outer_classname = "BigtableServiceMessagesProto"; @@ -35,6 +36,11 @@ message ReadRowsRequest { // A range of rows from which to read. RowRange row_range = 3; + + // A set of rows from which to read. Entries need not be in order, and will + // be deduplicated before reading. + // The total serialized size of the set must not exceed 1MB. + RowSet row_set = 8; } // The filter to apply to the contents of the specified row(s). If unset, @@ -124,6 +130,37 @@ message MutateRowRequest { repeated Mutation mutations = 3; } +// Request message for BigtableService.MutateRows. +message MutateRowsRequest { + message Entry { + // The key of the row to which the `mutations` should be applied. + bytes row_key = 1; + + // Changes to be atomically applied to the specified row. Mutations are + // applied in order, meaning that earlier mutations can be masked by + // later ones. + // At least one mutation must be specified. + repeated Mutation mutations = 2; + } + + // The unique name of the table to which the mutations should be applied. + string table_name = 1; + + // The row keys/mutations to be applied in bulk. + // Each entry is applied as an atomic mutation, but the entries may be + // applied in arbitrary order (even between entries for the same row). + // At least one entry must be specified, and in total the entries may + // contain at most 100000 mutations. + repeated Entry entries = 2; +} + +// Response message for BigtableService.MutateRows. +message MutateRowsResponse { + // The results for each Entry from the request, presented in the order + // in which the entries were originally given. + repeated google.rpc.Status statuses = 1; +} + // Request message for BigtableService.CheckAndMutateRowRequest message CheckAndMutateRowRequest { // The unique name of the table to which the conditional mutation should be diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py index fa8b7cf..4106aab 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py @@ -21,7 +21,7 @@ name='google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto', package='google.bigtable.admin.cluster.v1', syntax='proto3', - serialized_pb=b'\n\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3' + serialized_pb=b'\n&google/bigtable/v1/bigtable_data.proto\x12\x12google.bigtable.v1\"@\n\x03Row\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12,\n\x08\x66\x61milies\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Family\"C\n\x06\x46\x61mily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Column\"D\n\x06\x43olumn\x12\x11\n\tqualifier\x18\x01 \x01(\x0c\x12\'\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32\x18.google.bigtable.v1.Cell\"?\n\x04\x43\x65ll\x12\x18\n\x10timestamp_micros\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06labels\x18\x03 \x03(\t\".\n\x08RowRange\x12\x11\n\tstart_key\x18\x02 \x01(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\"L\n\x06RowSet\x12\x10\n\x08row_keys\x18\x01 \x03(\x0c\x12\x30\n\nrow_ranges\x18\x02 \x03(\x0b\x32\x1c.google.bigtable.v1.RowRange\"\xd6\x01\n\x0b\x43olumnRange\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12#\n\x19start_qualifier_inclusive\x18\x02 \x01(\x0cH\x00\x12#\n\x19start_qualifier_exclusive\x18\x03 \x01(\x0cH\x00\x12!\n\x17\x65nd_qualifier_inclusive\x18\x04 \x01(\x0cH\x01\x12!\n\x17\x65nd_qualifier_exclusive\x18\x05 \x01(\x0cH\x01\x42\x11\n\x0fstart_qualifierB\x0f\n\rend_qualifier\"N\n\x0eTimestampRange\x12\x1e\n\x16start_timestamp_micros\x18\x01 \x01(\x03\x12\x1c\n\x14\x65nd_timestamp_micros\x18\x02 \x01(\x03\"\xa8\x01\n\nValueRange\x12\x1f\n\x15start_value_inclusive\x18\x01 \x01(\x0cH\x00\x12\x1f\n\x15start_value_exclusive\x18\x02 \x01(\x0cH\x00\x12\x1d\n\x13\x65nd_value_inclusive\x18\x03 \x01(\x0cH\x01\x12\x1d\n\x13\x65nd_value_exclusive\x18\x04 \x01(\x0cH\x01\x42\r\n\x0bstart_valueB\x0b\n\tend_value\"\xdf\x08\n\tRowFilter\x12\x34\n\x05\x63hain\x18\x01 \x01(\x0b\x32#.google.bigtable.v1.RowFilter.ChainH\x00\x12>\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3' ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -221,6 +221,44 @@ ) +_ROWSET = _descriptor.Descriptor( + name='RowSet', + full_name='google.bigtable.v1.RowSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='row_keys', full_name='google.bigtable.v1.RowSet.row_keys', index=0, + number=1, type=12, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='row_ranges', full_name='google.bigtable.v1.RowSet.row_ranges', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=380, + serialized_end=456, +) + + _COLUMNRANGE = _descriptor.Descriptor( name='ColumnRange', full_name='google.bigtable.v1.ColumnRange', @@ -281,8 +319,8 @@ name='end_qualifier', full_name='google.bigtable.v1.ColumnRange.end_qualifier', index=1, containing_type=None, fields=[]), ], - serialized_start=381, - serialized_end=595, + serialized_start=459, + serialized_end=673, ) @@ -319,8 +357,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=597, - serialized_end=675, + serialized_start=675, + serialized_end=753, ) @@ -377,8 +415,8 @@ name='end_value', full_name='google.bigtable.v1.ValueRange.end_value', index=1, containing_type=None, fields=[]), ], - serialized_start=678, - serialized_end=846, + serialized_start=756, + serialized_end=924, ) @@ -408,8 +446,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1665, - serialized_end=1720, + serialized_start=1743, + serialized_end=1798, ) _ROWFILTER_INTERLEAVE = _descriptor.Descriptor( @@ -438,8 +476,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1722, - serialized_end=1782, + serialized_start=1800, + serialized_end=1860, ) _ROWFILTER_CONDITION = _descriptor.Descriptor( @@ -482,8 +520,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1785, - serialized_end=1958, + serialized_start=1863, + serialized_end=2036, ) _ROWFILTER = _descriptor.Descriptor( @@ -641,8 +679,8 @@ name='filter', full_name='google.bigtable.v1.RowFilter.filter', index=0, containing_type=None, fields=[]), ], - serialized_start=849, - serialized_end=1968, + serialized_start=927, + serialized_end=2046, ) @@ -693,8 +731,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2266, - serialized_end=2363, + serialized_start=2344, + serialized_end=2441, ) _MUTATION_DELETEFROMCOLUMN = _descriptor.Descriptor( @@ -737,8 +775,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2365, - serialized_end=2486, + serialized_start=2443, + serialized_end=2564, ) _MUTATION_DELETEFROMFAMILY = _descriptor.Descriptor( @@ -767,8 +805,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2488, - serialized_end=2527, + serialized_start=2566, + serialized_end=2605, ) _MUTATION_DELETEFROMROW = _descriptor.Descriptor( @@ -790,8 +828,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=2529, - serialized_end=2544, + serialized_start=2607, + serialized_end=2622, ) _MUTATION = _descriptor.Descriptor( @@ -844,8 +882,8 @@ name='mutation', full_name='google.bigtable.v1.Mutation.mutation', index=0, containing_type=None, fields=[]), ], - serialized_start=1971, - serialized_end=2556, + serialized_start=2049, + serialized_end=2634, ) @@ -899,13 +937,14 @@ name='rule', full_name='google.bigtable.v1.ReadModifyWriteRule.rule', index=0, containing_type=None, fields=[]), ], - serialized_start=2559, - serialized_end=2687, + serialized_start=2637, + serialized_end=2765, ) _ROW.fields_by_name['families'].message_type = _FAMILY _FAMILY.fields_by_name['columns'].message_type = _COLUMN _COLUMN.fields_by_name['cells'].message_type = _CELL +_ROWSET.fields_by_name['row_ranges'].message_type = _ROWRANGE _COLUMNRANGE.oneofs_by_name['start_qualifier'].fields.append( _COLUMNRANGE.fields_by_name['start_qualifier_inclusive']) _COLUMNRANGE.fields_by_name['start_qualifier_inclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['start_qualifier'] @@ -1033,6 +1072,7 @@ DESCRIPTOR.message_types_by_name['Column'] = _COLUMN DESCRIPTOR.message_types_by_name['Cell'] = _CELL DESCRIPTOR.message_types_by_name['RowRange'] = _ROWRANGE +DESCRIPTOR.message_types_by_name['RowSet'] = _ROWSET DESCRIPTOR.message_types_by_name['ColumnRange'] = _COLUMNRANGE DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE DESCRIPTOR.message_types_by_name['ValueRange'] = _VALUERANGE @@ -1075,6 +1115,13 @@ )) _sym_db.RegisterMessage(RowRange) +RowSet = _reflection.GeneratedProtocolMessageType('RowSet', (_message.Message,), dict( + DESCRIPTOR = _ROWSET, + __module__ = 'google.bigtable.v1.bigtable_data_pb2' + # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowSet) + )) +_sym_db.RegisterMessage(RowSet) + ColumnRange = _reflection.GeneratedProtocolMessageType('ColumnRange', (_message.Message,), dict( DESCRIPTOR = _COLUMNRANGE, __module__ = 'google.bigtable.v1.bigtable_data_pb2' diff --git a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py index e676bef..29d1d34 100644 --- a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py @@ -12,15 +12,16 @@ from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 +from google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='google/bigtable/v1/bigtable_service_messages.proto', package='google.bigtable.v1', syntax='proto3', - serialized_pb=b'\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\"\xdc\x01\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3' + serialized_pb=b'\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x17google/rpc/status.proto\"\x8b\x02\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x07row_set\x18\x08 \x01(\x0b\x32\x1a.google.bigtable.v1.RowSetH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xb0\x01\n\x11MutateRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12<\n\x07\x65ntries\x18\x02 \x03(\x0b\x32+.google.bigtable.v1.MutateRowsRequest.Entry\x1aI\n\x05\x45ntry\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12/\n\tmutations\x18\x02 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\":\n\x12MutateRowsResponse\x12$\n\x08statuses\x18\x01 \x03(\x0b\x32\x12.google.rpc.Status\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3' , - dependencies=[google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,]) + dependencies=[google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_rpc_dot_status__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -55,21 +56,28 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='filter', full_name='google.bigtable.v1.ReadRowsRequest.filter', index=3, + name='row_set', full_name='google.bigtable.v1.ReadRowsRequest.row_set', index=3, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='filter', full_name='google.bigtable.v1.ReadRowsRequest.filter', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='allow_row_interleaving', full_name='google.bigtable.v1.ReadRowsRequest.allow_row_interleaving', index=4, + name='allow_row_interleaving', full_name='google.bigtable.v1.ReadRowsRequest.allow_row_interleaving', index=5, number=6, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='num_rows_limit', full_name='google.bigtable.v1.ReadRowsRequest.num_rows_limit', index=5, + name='num_rows_limit', full_name='google.bigtable.v1.ReadRowsRequest.num_rows_limit', index=6, number=7, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, @@ -90,8 +98,8 @@ name='target', full_name='google.bigtable.v1.ReadRowsRequest.target', index=0, containing_type=None, fields=[]), ], - serialized_start=115, - serialized_end=335, + serialized_start=140, + serialized_end=407, ) @@ -138,8 +146,8 @@ name='chunk', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.chunk', index=0, containing_type=None, fields=[]), ], - serialized_start=435, - serialized_end=546, + serialized_start=507, + serialized_end=618, ) _READROWSRESPONSE = _descriptor.Descriptor( @@ -175,8 +183,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=338, - serialized_end=546, + serialized_start=410, + serialized_end=618, ) @@ -206,8 +214,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=548, - serialized_end=590, + serialized_start=620, + serialized_end=662, ) @@ -244,8 +252,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=592, - serialized_end=654, + serialized_start=664, + serialized_end=726, ) @@ -289,8 +297,114 @@ extension_ranges=[], oneofs=[ ], - serialized_start=656, - serialized_end=760, + serialized_start=728, + serialized_end=832, +) + + +_MUTATEROWSREQUEST_ENTRY = _descriptor.Descriptor( + name='Entry', + full_name='google.bigtable.v1.MutateRowsRequest.Entry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='row_key', full_name='google.bigtable.v1.MutateRowsRequest.Entry.row_key', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mutations', full_name='google.bigtable.v1.MutateRowsRequest.Entry.mutations', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=938, + serialized_end=1011, +) + +_MUTATEROWSREQUEST = _descriptor.Descriptor( + name='MutateRowsRequest', + full_name='google.bigtable.v1.MutateRowsRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='table_name', full_name='google.bigtable.v1.MutateRowsRequest.table_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='entries', full_name='google.bigtable.v1.MutateRowsRequest.entries', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_MUTATEROWSREQUEST_ENTRY, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=835, + serialized_end=1011, +) + + +_MUTATEROWSRESPONSE = _descriptor.Descriptor( + name='MutateRowsResponse', + full_name='google.bigtable.v1.MutateRowsResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='statuses', full_name='google.bigtable.v1.MutateRowsResponse.statuses', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1013, + serialized_end=1071, ) @@ -348,8 +462,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=763, - serialized_end=992, + serialized_start=1074, + serialized_end=1303, ) @@ -379,8 +493,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=994, - serialized_end=1048, + serialized_start=1305, + serialized_end=1359, ) @@ -424,11 +538,12 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1050, - serialized_end=1170, + serialized_start=1361, + serialized_end=1481, ) _READROWSREQUEST.fields_by_name['row_range'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWRANGE +_READROWSREQUEST.fields_by_name['row_set'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWSET _READROWSREQUEST.fields_by_name['filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER _READROWSREQUEST.oneofs_by_name['target'].fields.append( _READROWSREQUEST.fields_by_name['row_key']) @@ -436,6 +551,9 @@ _READROWSREQUEST.oneofs_by_name['target'].fields.append( _READROWSREQUEST.fields_by_name['row_range']) _READROWSREQUEST.fields_by_name['row_range'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] +_READROWSREQUEST.oneofs_by_name['target'].fields.append( + _READROWSREQUEST.fields_by_name['row_set']) +_READROWSREQUEST.fields_by_name['row_set'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] _READROWSRESPONSE_CHUNK.fields_by_name['row_contents'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._FAMILY _READROWSRESPONSE_CHUNK.containing_type = _READROWSRESPONSE _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( @@ -449,6 +567,10 @@ _READROWSRESPONSE_CHUNK.fields_by_name['commit_row'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] _READROWSRESPONSE.fields_by_name['chunks'].message_type = _READROWSRESPONSE_CHUNK _MUTATEROWREQUEST.fields_by_name['mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION +_MUTATEROWSREQUEST_ENTRY.fields_by_name['mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION +_MUTATEROWSREQUEST_ENTRY.containing_type = _MUTATEROWSREQUEST +_MUTATEROWSREQUEST.fields_by_name['entries'].message_type = _MUTATEROWSREQUEST_ENTRY +_MUTATEROWSRESPONSE.fields_by_name['statuses'].message_type = google_dot_rpc_dot_status__pb2._STATUS _CHECKANDMUTATEROWREQUEST.fields_by_name['predicate_filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER _CHECKANDMUTATEROWREQUEST.fields_by_name['true_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION _CHECKANDMUTATEROWREQUEST.fields_by_name['false_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION @@ -458,6 +580,8 @@ DESCRIPTOR.message_types_by_name['SampleRowKeysRequest'] = _SAMPLEROWKEYSREQUEST DESCRIPTOR.message_types_by_name['SampleRowKeysResponse'] = _SAMPLEROWKEYSRESPONSE DESCRIPTOR.message_types_by_name['MutateRowRequest'] = _MUTATEROWREQUEST +DESCRIPTOR.message_types_by_name['MutateRowsRequest'] = _MUTATEROWSREQUEST +DESCRIPTOR.message_types_by_name['MutateRowsResponse'] = _MUTATEROWSRESPONSE DESCRIPTOR.message_types_by_name['CheckAndMutateRowRequest'] = _CHECKANDMUTATEROWREQUEST DESCRIPTOR.message_types_by_name['CheckAndMutateRowResponse'] = _CHECKANDMUTATEROWRESPONSE DESCRIPTOR.message_types_by_name['ReadModifyWriteRowRequest'] = _READMODIFYWRITEROWREQUEST @@ -505,6 +629,28 @@ )) _sym_db.RegisterMessage(MutateRowRequest) +MutateRowsRequest = _reflection.GeneratedProtocolMessageType('MutateRowsRequest', (_message.Message,), dict( + + Entry = _reflection.GeneratedProtocolMessageType('Entry', (_message.Message,), dict( + DESCRIPTOR = _MUTATEROWSREQUEST_ENTRY, + __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' + # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsRequest.Entry) + )) + , + DESCRIPTOR = _MUTATEROWSREQUEST, + __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' + # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsRequest) + )) +_sym_db.RegisterMessage(MutateRowsRequest) +_sym_db.RegisterMessage(MutateRowsRequest.Entry) + +MutateRowsResponse = _reflection.GeneratedProtocolMessageType('MutateRowsResponse', (_message.Message,), dict( + DESCRIPTOR = _MUTATEROWSRESPONSE, + __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' + # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsResponse) + )) +_sym_db.RegisterMessage(MutateRowsResponse) + CheckAndMutateRowRequest = _reflection.GeneratedProtocolMessageType('CheckAndMutateRowRequest', (_message.Message,), dict( DESCRIPTOR = _CHECKANDMUTATEROWREQUEST, __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index bd64308..189b3a1 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -21,7 +21,7 @@ name='google/bigtable/v1/bigtable_service.proto', package='google.bigtable.v1', syntax='proto3', - serialized_pb=b'\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xb0\x07\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3' + serialized_pb=b'\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xdd\x08\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xaa\x01\n\nMutateRows\x12%.google.bigtable.v1.MutateRowsRequest\x1a&.google.bigtable.v1.MutateRowsResponse\"M\x82\xd3\xe4\x93\x02G\"B/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}:mutateRows:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3' , dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -51,6 +51,9 @@ def SampleRowKeys(self, request, context): def MutateRow(self, request, context): raise NotImplementedError() @abc.abstractmethod + def MutateRows(self, request, context): + raise NotImplementedError() + @abc.abstractmethod def CheckAndMutateRow(self, request, context): raise NotImplementedError() @abc.abstractmethod @@ -81,6 +84,10 @@ def MutateRow(self, request): raise NotImplementedError() MutateRow.async = None @abc.abstractmethod + def MutateRows(self, request): + raise NotImplementedError() + MutateRows.async = None + @abc.abstractmethod def CheckAndMutateRow(self, request): raise NotImplementedError() CheckAndMutateRow.async = None @@ -98,6 +105,8 @@ def early_adopter_create_BigtableService_server(servicer, port, private_key=None import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 method_service_descriptions = { "CheckAndMutateRow": alpha_utilities.unary_unary_service_description( @@ -110,6 +119,11 @@ def early_adopter_create_BigtableService_server(servicer, port, private_key=None gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, google.protobuf.empty_pb2.Empty.SerializeToString, ), + "MutateRows": alpha_utilities.unary_unary_service_description( + servicer.MutateRows, + gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.FromString, + gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.SerializeToString, + ), "ReadModifyWriteRow": alpha_utilities.unary_unary_service_description( servicer.ReadModifyWriteRow, gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, @@ -137,6 +151,8 @@ def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=N import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 method_invocation_descriptions = { "CheckAndMutateRow": alpha_utilities.unary_unary_invocation_description( @@ -147,6 +163,10 @@ def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=N gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, google.protobuf.empty_pb2.Empty.FromString, ), + "MutateRows": alpha_utilities.unary_unary_invocation_description( + gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.SerializeToString, + gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.FromString, + ), "ReadModifyWriteRow": alpha_utilities.unary_unary_invocation_description( gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, @@ -175,6 +195,9 @@ def SampleRowKeys(self, request, context): def MutateRow(self, request, context): raise NotImplementedError() @abc.abstractmethod + def MutateRows(self, request, context): + raise NotImplementedError() + @abc.abstractmethod def CheckAndMutateRow(self, request, context): raise NotImplementedError() @abc.abstractmethod @@ -195,6 +218,10 @@ def MutateRow(self, request, timeout): raise NotImplementedError() MutateRow.future = None @abc.abstractmethod + def MutateRows(self, request, timeout): + raise NotImplementedError() + MutateRows.future = None + @abc.abstractmethod def CheckAndMutateRow(self, request, timeout): raise NotImplementedError() CheckAndMutateRow.future = None @@ -213,10 +240,13 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 request_deserializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.FromString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, @@ -224,6 +254,7 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa response_serializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, @@ -231,6 +262,7 @@ def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, defa method_implementations = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): face_utilities.unary_unary_inline(servicer.CheckAndMutateRow), ('google.bigtable.v1.BigtableService', 'MutateRow'): face_utilities.unary_unary_inline(servicer.MutateRow), + ('google.bigtable.v1.BigtableService', 'MutateRows'): face_utilities.unary_unary_inline(servicer.MutateRows), ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): face_utilities.unary_unary_inline(servicer.ReadModifyWriteRow), ('google.bigtable.v1.BigtableService', 'ReadRows'): face_utilities.unary_stream_inline(servicer.ReadRows), ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): face_utilities.unary_stream_inline(servicer.SampleRowKeys), @@ -248,10 +280,13 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 + import gcloud_bigtable._generated.bigtable_service_messages_pb2 import gcloud_bigtable._generated.bigtable_data_pb2 request_serializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, + ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, @@ -259,6 +294,7 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No response_deserializers = { ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.FromString, + ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.FromString, ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, @@ -266,6 +302,7 @@ def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=No cardinalities = { 'CheckAndMutateRow': cardinality.Cardinality.UNARY_UNARY, 'MutateRow': cardinality.Cardinality.UNARY_UNARY, + 'MutateRows': cardinality.Cardinality.UNARY_UNARY, 'ReadModifyWriteRow': cardinality.Cardinality.UNARY_UNARY, 'ReadRows': cardinality.Cardinality.UNARY_STREAM, 'SampleRowKeys': cardinality.Cardinality.UNARY_STREAM, From 206ee5c16b00d426f8e7e0ba648152aa0ecce388 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 19 Feb 2016 23:37:44 -0800 Subject: [PATCH 50/65] Upstream updates. --- gcloud_bigtable/happybase/connection.py | 111 +++++++++---------- gcloud_bigtable/happybase/test_connection.py | 21 ++-- gcloud_bigtable/table.py | 4 +- 3 files changed, 71 insertions(+), 65 deletions(-) diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 3d105e6..092b74e 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -84,56 +84,6 @@ def _get_cluster(timeout=None): return clusters[0] -def _parse_family_option(option): - """Parses a column family option into a garbage collection rule. - - .. note:: - - If ``option`` is not a dictionary, the type is not checked. - If ``option`` is :data:`None`, there is nothing to do, since this - is the correct output. - - :type option: :class:`dict`, - :data:`NoneType `, - :class:`.GarbageCollectionRule` - :param option: A column family option passes as a dictionary value in - :meth:`Connection.create_table`. - - :rtype: :class:`.GarbageCollectionRule` - :returns: A garbage collection rule parsed from the input. - :raises: :class:`ValueError ` if ``option`` is a - dictionary but keys other than ``max_versions`` and - ``time_to_live`` are used. - """ - result = option - if isinstance(result, dict): - # pylint: disable=unneeded-not - if not set(result.keys()) <= set(['max_versions', 'time_to_live']): - raise ValueError('Cloud Bigtable only supports max_versions and ' - 'time_to_live column family settings', - 'Received', result.keys()) - # pylint: enable=unneeded-not - - max_num_versions = result.get('max_versions') - max_age = None - if 'time_to_live' in result: - max_age = datetime.timedelta(seconds=result['time_to_live']) - - if len(result) == 0: - result = None - elif len(result) == 1: - if max_num_versions is None: - result = MaxAgeGCRule(max_age) - else: - result = MaxVersionsGCRule(max_num_versions) - else: # By our check above we know this means len(result) == 2. - rule1 = MaxAgeGCRule(max_age) - rule2 = MaxVersionsGCRule(max_num_versions) - result = GCRuleIntersection(rules=[rule1, rule2]) - - return result - - class Connection(object): """Connection to Cloud Bigtable backend. @@ -183,6 +133,8 @@ class Connection(object): parameters are specified with a value other than the defaults. """ + _cluster = None + def __init__(self, timeout=None, autoconnect=True, table_prefix=None, table_prefix_separator='_', cluster=None, **kwargs): self._handle_legacy_args(kwargs) @@ -254,12 +206,7 @@ def close(self): self._cluster._client.stop() def __del__(self): - try: - self._initialized - except AttributeError: - # Failure from constructor - return - else: + if self._cluster is not None: self.close() def _table_name(self, name): @@ -377,6 +324,8 @@ def create_table(self, name, families): # Parse all keys before making any API requests. gc_rule_dict = {} for column_family_name, option in families.items(): + if isinstance(column_family_name, six.binary_type): + column_family_name = column_family_name.decode('utf-8') if column_family_name.endswith(':'): column_family_name = column_family_name[:-1] gc_rule_dict[column_family_name] = _parse_family_option(option) @@ -462,3 +411,53 @@ def compact_table(self, name, major=False): """ raise NotImplementedError('The Cloud Bigtable API does not support ' 'compacting a table.') + + +def _parse_family_option(option): + """Parses a column family option into a garbage collection rule. + + .. note:: + + If ``option`` is not a dictionary, the type is not checked. + If ``option`` is :data:`None`, there is nothing to do, since this + is the correct output. + + :type option: :class:`dict`, + :data:`NoneType `, + :class:`.GarbageCollectionRule` + :param option: A column family option passes as a dictionary value in + :meth:`Connection.create_table`. + + :rtype: :class:`.GarbageCollectionRule` + :returns: A garbage collection rule parsed from the input. + :raises: :class:`ValueError ` if ``option`` is a + dictionary but keys other than ``max_versions`` and + ``time_to_live`` are used. + """ + result = option + if isinstance(result, dict): + # pylint: disable=unneeded-not + if not set(result.keys()) <= set(['max_versions', 'time_to_live']): + raise ValueError('Cloud Bigtable only supports max_versions and ' + 'time_to_live column family settings', + 'Received', result.keys()) + # pylint: enable=unneeded-not + + max_num_versions = result.get('max_versions') + max_age = None + if 'time_to_live' in result: + max_age = datetime.timedelta(seconds=result['time_to_live']) + + if len(result) == 0: + result = None + elif len(result) == 1: + if max_num_versions is None: + result = MaxAgeGCRule(max_age) + else: + result = MaxVersionsGCRule(max_num_versions) + else: # By our check above we know this means len(result) == 2. + rule1 = MaxAgeGCRule(max_age) + rule2 = MaxVersionsGCRule(max_num_versions) + result = GCRuleIntersection(rules=[rule1, rule2]) + + return result diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index 34475bc..763dedb 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -277,7 +277,7 @@ def test___del__bad_initialization(self): cluster = _Cluster() # Avoid implicit environ check. connection = self._makeOne(autoconnect=False, cluster=cluster) # Fake that initialization failed. - del connection._initialized + del connection._cluster self.assertEqual(cluster._client.stop_calls, 0) connection.__del__() @@ -396,12 +396,15 @@ def test_create_table(self): name = 'table-name' col_fam1 = 'cf1' col_fam_option1 = object() - col_fam2 = 'cf2' + col_fam2 = u'cf2' col_fam_option2 = object() + col_fam3 = b'cf3' + col_fam_option3 = object() families = { col_fam1: col_fam_option1, # A trailing colon is also allowed. - col_fam2 + ':': col_fam_option2, + col_fam2 + u':': col_fam_option2, + col_fam3 + b':': col_fam_option3, } table_instances = [] col_fam_instances = [] @@ -419,10 +422,11 @@ def test_create_table(self): # Check if our mock was called twice, but we don't know the order. mock_called = mock_parse_family_option.called_args - self.assertEqual(len(mock_called), 2) - self.assertEqual([len(args) for args in mock_called], [1, 1]) - self.assertEqual(set(mock_called[0] + mock_called[1]), - set([col_fam_option1, col_fam_option2])) + self.assertEqual(len(mock_called), 3) + self.assertEqual([len(args) for args in mock_called], [1, 1, 1]) + self.assertEqual( + set(mock_called[0] + mock_called[1] + mock_called[2]), + set([col_fam_option1, col_fam_option2, col_fam_option3])) # We expect two column family instances created, but don't know the # order due to non-deterministic dict.items(). @@ -433,6 +437,9 @@ def test_create_table(self): self.assertEqual(col_fam_instances[1].column_family_id, col_fam2) self.assertEqual(col_fam_instances[1].gc_rule, mock_gc_rule) self.assertEqual(col_fam_instances[1].create_calls, 1) + self.assertEqual(col_fam_instances[2].column_family_id, col_fam3) + self.assertEqual(col_fam_instances[2].gc_rule, mock_gc_rule) + self.assertEqual(col_fam_instances[2].create_calls, 1) def test_create_table_bad_type(self): cluster = _Cluster() # Avoid implicit environ check. diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index f45ddb7..104c00c 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -15,14 +15,14 @@ """User friendly container for Google Cloud Bigtable Table.""" +from gcloud_bigtable._non_upstream_helpers import _to_bytes from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_service_messages_pb2 as data_messages_pb2) from gcloud_bigtable._generated import ( bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._non_upstream_helpers import _to_bytes -from gcloud_bigtable.column_family import ColumnFamily from gcloud_bigtable.column_family import _gc_rule_from_pb +from gcloud_bigtable.column_family import ColumnFamily from gcloud_bigtable.row import Row from gcloud_bigtable.row_data import PartialRowData from gcloud_bigtable.row_data import PartialRowsData From 2267f14311714bc72834b0e2909780c5a3722a37 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 19 Feb 2016 23:39:51 -0800 Subject: [PATCH 51/65] Some simple updates. --- gcloud_bigtable/happybase/table.py | 2 -- gcloud_bigtable/happybase/test_table.py | 1 - 2 files changed, 3 deletions(-) diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index 66f51da..54ac7f3 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -660,8 +660,6 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, :raises: :class:`ValueError ` if ``batch_size`` or ``scan_batching`` are used, or if ``limit`` is set but non-positive, or if row prefix is used with row start/stop, - :class:`NotImplementedError ` - temporarily until the method is implemented, :class:`TypeError ` if a string ``filter`` is used. """ diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py index 65774c1..3db5fc2 100644 --- a/gcloud_bigtable/happybase/test_table.py +++ b/gcloud_bigtable/happybase/test_table.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import unittest2 From e314a1ed3b600aa0a55eb3320d38cc5b02f8a8ad Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Fri, 19 Feb 2016 23:59:38 -0800 Subject: [PATCH 52/65] Updating for lint and new code in oauth2client. --- gcloud_bigtable/_non_upstream_helpers.py | 8 ++--- gcloud_bigtable/happybase/connection.py | 2 -- gcloud_bigtable/test__non_upstream_helpers.py | 30 +++++++++---------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 3d38b2d..4d053e7 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -27,8 +27,8 @@ import six from oauth2client.client import GoogleCredentials -from oauth2client.client import SignedJwtAssertionCredentials from oauth2client.client import _get_application_default_credential_from_file +from oauth2client.service_account import ServiceAccountCredentials try: from google.appengine.api import app_identity @@ -212,10 +212,8 @@ def from_service_account_p12(cls, client_email, private_key_path, :rtype: :class:`_FactoryMixin` :returns: The client created with the retrieved P12 credentials. """ - credentials = SignedJwtAssertionCredentials( - service_account_name=client_email, - private_key=_get_contents(private_key_path), - scope=None) + credentials = ServiceAccountCredentials.from_p12_keyfile( + client_email, private_key_path) kwargs['credentials'] = credentials return cls(*args, **kwargs) diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index 092b74e..fce97de 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -436,12 +436,10 @@ def _parse_family_option(option): """ result = option if isinstance(result, dict): - # pylint: disable=unneeded-not if not set(result.keys()) <= set(['max_versions', 'time_to_live']): raise ValueError('Cloud Bigtable only supports max_versions and ' 'time_to_live column family settings', 'Received', result.keys()) - # pylint: enable=unneeded-not max_num_versions = result.get('max_versions') max_age = None diff --git a/gcloud_bigtable/test__non_upstream_helpers.py b/gcloud_bigtable/test__non_upstream_helpers.py index 31a19e8..c619f3c 100644 --- a/gcloud_bigtable/test__non_upstream_helpers.py +++ b/gcloud_bigtable/test__non_upstream_helpers.py @@ -273,7 +273,6 @@ def test_from_service_account_json(self): get_adc.check_called(self, [(json_credentials_path,)]) def test_from_service_account_p12(self): - from gcloud_bigtable._testing import _MockCalled from gcloud_bigtable._testing import _MockWithAttachedMethods from gcloud_bigtable._testing import _Monkey from gcloud_bigtable import _non_upstream_helpers as MUT @@ -281,30 +280,20 @@ def test_from_service_account_p12(self): klass = self._getTargetClass() scoped_creds = object() credentials = _MockWithAttachedMethods(scoped_creds) - signed_creds = _MockCalled(credentials) + signed_creds = _MockServiceAccountCredentials(credentials) - private_key = 'PRIVATE_KEY' - mock_get_contents = _MockCalled(private_key) client_email = 'CLIENT_EMAIL' private_key_path = 'PRIVATE_KEY_PATH' - with _Monkey(MUT, SignedJwtAssertionCredentials=signed_creds, - _get_contents=mock_get_contents): + with _Monkey(MUT, ServiceAccountCredentials=signed_creds): client = klass.from_service_account_p12( client_email, private_key_path, project=PROJECT) + self.assertEqual(signed_creds.p12_called, + [(client_email, private_key_path)]) self.assertEqual(client.project, PROJECT) self.assertTrue(client._credentials is credentials) self.assertEqual(credentials._called, []) - # SignedJwtAssertionCredentials() called with only kwargs - signed_creds_kw = { - 'private_key': private_key, - 'service_account_name': client_email, - 'scope': None, - } - signed_creds.check_called(self, [()], [signed_creds_kw]) - # Load private key (via _get_contents) from the key path. - mock_get_contents.check_called(self, [(private_key_path,)]) class Test__timestamp_to_microseconds(unittest2.TestCase): @@ -414,3 +403,14 @@ def test_it(self): microseconds=414000) result = self._callFUT(offset) self.assertEqual(result, 1.414) + + +class _MockServiceAccountCredentials(object): + + def __init__(self, result): + self.result = result + self.p12_called = [] + + def from_p12_keyfile(self, client_email, path): + self.p12_called.append((client_email, path)) + return self.result From 8d91feb7daf4ccac7818d468249c4c95f0e14a5e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sat, 20 Feb 2016 00:17:15 -0800 Subject: [PATCH 53/65] Moving code (based on upstream). --- gcloud_bigtable/happybase/table.py | 91 +++++++++++++++--------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index 54ac7f3..dfb01b9 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -94,52 +94,6 @@ def make_ordered_row(sorted_columns, include_timestamp): 'helper can not be implemented.', 'Called with', sorted_columns, include_timestamp) - -def _gc_rule_to_dict(gc_rule): - """Converts garbage collection rule to dictionary if possible. - - This is in place to support dictionary values was was done - in HappyBase, which has somewhat different garbage collection rule - settings for column families. - - Only does this if the garbage collection rule is: - - * :class:`.MaxAgeGCRule` - * :class:`.MaxVersionsGCRule` - * Composite :class:`.GCRuleIntersection` with two rules, one each - of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` - - Otherwise, just returns the input without change. - - :type gc_rule: :data:`NoneType `, - :class:`.GarbageCollectionRule` - :param gc_rule: A garbage collection rule to convert to a dictionary - (if possible). - - :rtype: dict or :class:`.GarbageCollectionRule` - :returns: The converted garbage collection rule. - """ - result = gc_rule - if gc_rule is None: - result = {} - elif isinstance(gc_rule, MaxAgeGCRule): - result = {'time_to_live': gc_rule.max_age.total_seconds()} - elif isinstance(gc_rule, MaxVersionsGCRule): - result = {'max_versions': gc_rule.max_num_versions} - elif isinstance(gc_rule, GCRuleIntersection): - if len(gc_rule.rules) == 2: - rule1, rule2 = gc_rule.rules - if (isinstance(rule1, _SIMPLE_GC_RULES) and - isinstance(rule2, _SIMPLE_GC_RULES)): - rule1 = _gc_rule_to_dict(rule1) - rule2 = _gc_rule_to_dict(rule2) - key1, = rule1.keys() - key2, = rule2.keys() - if key1 != key2: - result = {key1: rule1[key1], key2: rule2[key2]} - return result - - def _convert_to_time_range(timestamp=None): """Create a timestamp range from an HBase / HappyBase timestamp. @@ -914,3 +868,48 @@ def counter_dec(self, row, column, value=1): :returns: Counter value after decrementing. """ return self.counter_inc(row, column, -value) + + +def _gc_rule_to_dict(gc_rule): + """Converts garbage collection rule to dictionary if possible. + + This is in place to support dictionary values as was done + in HappyBase, which has somewhat different garbage collection rule + settings for column families. + + Only does this if the garbage collection rule is: + + * :class:`.MaxAgeGCRule` + * :class:`.MaxVersionsGCRule` + * Composite :class:`.GCRuleIntersection` with two rules, one each + of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` + + Otherwise, just returns the input without change. + + :type gc_rule: :data:`NoneType `, + :class:`.GarbageCollectionRule` + :param gc_rule: A garbage collection rule to convert to a dictionary + (if possible). + + :rtype: dict or :class:`.GarbageCollectionRule` + :returns: The converted garbage collection rule. + """ + result = gc_rule + if gc_rule is None: + result = {} + elif isinstance(gc_rule, MaxAgeGCRule): + result = {'time_to_live': gc_rule.max_age.total_seconds()} + elif isinstance(gc_rule, MaxVersionsGCRule): + result = {'max_versions': gc_rule.max_num_versions} + elif isinstance(gc_rule, GCRuleIntersection): + if len(gc_rule.rules) == 2: + rule1, rule2 = gc_rule.rules + if (isinstance(rule1, _SIMPLE_GC_RULES) and + isinstance(rule2, _SIMPLE_GC_RULES)): + rule1 = _gc_rule_to_dict(rule1) + rule2 = _gc_rule_to_dict(rule2) + key1, = rule1.keys() + key2, = rule2.keys() + if key1 != key2: + result = {key1: rule1[key1], key2: rule2[key2]} + return result From c1cd1be09edb51c70f5019e33b78083af85243b7 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 10:03:52 -0800 Subject: [PATCH 54/65] Upstream changes. --- gcloud_bigtable/happybase/batch.py | 19 +++++++++++++------ gcloud_bigtable/happybase/table.py | 12 ++++++++---- gcloud_bigtable/table.py | 20 +++++++------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index 0fbdc65..4fb15b8 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -16,15 +16,23 @@ import datetime +import warnings + import six from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable.row import TimestampRange +# BEGIN: Renames for upstream. +_datetime_from_microseconds = _microseconds_to_timestamp +# END: Renames for upstream. _WAL_SENTINEL = object() # Assumed granularity of timestamps in Cloud Bigtable. _ONE_MILLISECOND = datetime.timedelta(microseconds=1000) +_WARN = warnings.warn +_WAL_WARNING = ('The wal argument (Write-Ahead-Log) is not ' + 'supported by Cloud Bigtable.') def _get_column_pairs(columns, require_qualifier=False): @@ -106,15 +114,12 @@ class Batch(object): is set and ``transaction=True``. :class:`ValueError ` if ``batch_size`` is not positive. - :class:`ValueError ` if ``wal`` - is used. """ def __init__(self, table, timestamp=None, batch_size=None, transaction=False, wal=_WAL_SENTINEL): if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') + _WARN(_WAL_WARNING) if batch_size is not None: if transaction: @@ -125,10 +130,11 @@ def __init__(self, table, timestamp=None, batch_size=None, self._table = table self._batch_size = batch_size - # Timestamp is in milliseconds, convert to microseconds. self._timestamp = self._delete_range = None + + # Timestamp is in milliseconds, convert to microseconds. if timestamp is not None: - self._timestamp = _microseconds_to_timestamp(1000 * timestamp) + self._timestamp = _datetime_from_microseconds(1000 * timestamp) # For deletes, we get the very next timestamp (assuming timestamp # granularity is milliseconds). This is because HappyBase users # expect HBase deletes to go **up to** and **including** the @@ -136,6 +142,7 @@ def __init__(self, table, timestamp=None, batch_size=None, # final timestamp. next_timestamp = self._timestamp + _ONE_MILLISECOND self._delete_range = TimestampRange(end=next_timestamp) + self._transaction = transaction # Internal state for tracking mutations. diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index dfb01b9..dd6d8ea 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -349,7 +349,7 @@ class Table(object): def __init__(self, name, connection): self.name = name # This remains as legacy for HappyBase, but only the cluster - # from it is needed. + # from the connection is needed. self.connection = connection self._low_level_table = None if self.connection is not None: @@ -367,8 +367,10 @@ def families(self): for a column family. """ column_family_map = self._low_level_table.list_column_families() - return {col_fam: _gc_rule_to_dict(col_fam_obj.gc_rule) - for col_fam, col_fam_obj in column_family_map.items()} + result = {} + for col_fam, col_fam_obj in six.iteritems(column_family_map): + result[col_fam] = _gc_rule_to_dict(col_fam_obj.gc_rule) + return result def regions(self): """Retrieve the regions for this table. @@ -787,7 +789,7 @@ def counter_get(self, row, column): :returns: Counter value (after initializing / incrementing by 0). """ # Don't query directly, but increment with value=0 so that the counter - # is correctly initialised if didn't exist yet. + # is correctly initialized if didn't exist yet. return self.counter_inc(row, column, value=0) def counter_set(self, row, column, value=0): @@ -836,6 +838,8 @@ def counter_inc(self, row, column, value=1): :returns: Counter value after incrementing. """ row = self._low_level_table.row(row) + if isinstance(column, six.binary_type): + column = column.decode('utf-8') column_family_id, column_qualifier = column.split(':') row.increment_cell_value(column_family_id, column_qualifier, value) modified_cells = row.commit_modifications() diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py index 104c00c..28386ff 100644 --- a/gcloud_bigtable/table.py +++ b/gcloud_bigtable/table.py @@ -228,9 +228,7 @@ def read_row(self, row_key, filter_=None): :type row_key: bytes :param row_key: The key of the row to read from. - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` + :type filter_: :class:`.row.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the row. If unset, returns the entire row. @@ -272,13 +270,6 @@ def read_rows(self, start_key=None, end_key=None, The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - specified row(s). If unset, reads every column in - each row. - :type allow_row_interleaving: bool :param allow_row_interleaving: (Optional) By default, rows are read sequentially, producing results which @@ -300,6 +291,11 @@ def read_rows(self, start_key=None, end_key=None, more than N rows. However, only N ``commit_row`` chunks will be sent. + :type filter_: :class:`.row.RowFilter` + :param filter_: (Optional) The filter to apply to the contents of the + specified row(s). If unset, reads every column in + each row. + :rtype: :class:`.PartialRowsData` :returns: A :class:`.PartialRowsData` convenience wrapper for consuming the streamed results. @@ -372,9 +368,7 @@ def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, The range will not include ``end_key``. If left empty, will be interpreted as an infinite string. - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` + :type filter_: :class:`.row.RowFilter` :param filter_: (Optional) The filter to apply to the contents of the specified row(s). If unset, reads the entire table. From b2c61ee0d105013e4152711c8563cc3365ea58d7 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 10:18:10 -0800 Subject: [PATCH 55/65] Fixing some docstrings/tests broken by upstream. --- gcloud_bigtable/happybase/connection.py | 38 ++++++++++---------- gcloud_bigtable/happybase/test_batch.py | 12 ++++++- gcloud_bigtable/happybase/test_connection.py | 16 +++++++-- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py index fce97de..e82f600 100644 --- a/gcloud_bigtable/happybase/connection.py +++ b/gcloud_bigtable/happybase/connection.py @@ -15,9 +15,9 @@ """Google Cloud Bigtable HappyBase connection module.""" +import datetime import warnings -import datetime import six from gcloud_bigtable.client import Client @@ -128,9 +128,6 @@ class Connection(object): :type kwargs: dict :param kwargs: Remaining keyword arguments. Provided for HappyBase compatibility. - - :raises: :class:`ValueError ` if any of the unused - parameters are specified with a value other than the defaults. """ _cluster = None @@ -430,32 +427,33 @@ def _parse_family_option(option): :rtype: :class:`.GarbageCollectionRule` :returns: A garbage collection rule parsed from the input. - :raises: :class:`ValueError ` if ``option`` is a - dictionary but keys other than ``max_versions`` and - ``time_to_live`` are used. """ result = option if isinstance(result, dict): if not set(result.keys()) <= set(['max_versions', 'time_to_live']): - raise ValueError('Cloud Bigtable only supports max_versions and ' - 'time_to_live column family settings', - 'Received', result.keys()) + all_keys = ', '.join(repr(key) for key in result.keys()) + warning_msg = ('Cloud Bigtable only supports max_versions and ' + 'time_to_live column family settings. ' + 'Received: %s' % (all_keys,)) + _WARN(warning_msg) max_num_versions = result.get('max_versions') max_age = None if 'time_to_live' in result: max_age = datetime.timedelta(seconds=result['time_to_live']) - if len(result) == 0: - result = None - elif len(result) == 1: - if max_num_versions is None: - result = MaxAgeGCRule(max_age) + versions_rule = age_rule = None + if max_num_versions is not None: + versions_rule = MaxVersionsGCRule(max_num_versions) + if max_age is not None: + age_rule = MaxAgeGCRule(max_age) + + if versions_rule is None: + result = age_rule + else: + if age_rule is None: + result = versions_rule else: - result = MaxVersionsGCRule(max_num_versions) - else: # By our check above we know this means len(result) == 2. - rule1 = MaxAgeGCRule(max_age) - rule2 = MaxVersionsGCRule(max_num_versions) - result = GCRuleIntersection(rules=[rule1, rule2]) + result = GCRuleIntersection(rules=[age_rule, versions_rule]) return result diff --git a/gcloud_bigtable/happybase/test_batch.py b/gcloud_bigtable/happybase/test_batch.py index 7ccee33..88e2ebc 100644 --- a/gcloud_bigtable/happybase/test_batch.py +++ b/gcloud_bigtable/happybase/test_batch.py @@ -104,11 +104,21 @@ def test_constructor_explicit(self): self.assertEqual(batch._transaction, transaction) def test_constructor_with_non_default_wal(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import batch as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + table = object() wal = object() - with self.assertRaises(ValueError): + with _Monkey(MUT, _WARN=mock_warn): self._makeOne(table, wal=wal) + self.assertEqual(warned, [MUT._WAL_WARNING]) + def test_constructor_with_non_positive_batch_size(self): table = object() batch_size = -10 diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py index 763dedb..9aa2c13 100644 --- a/gcloud_bigtable/happybase/test_connection.py +++ b/gcloud_bigtable/happybase/test_connection.py @@ -87,9 +87,21 @@ def test_null(self): self.assertEqual(result, None) def test_dictionary_bad_key(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import connection as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + option = {'badkey': None} - with self.assertRaises(ValueError): - self._callFUT(option) + with _Monkey(MUT, _WARN=mock_warn): + result = self._callFUT(option) + + self.assertEqual(result, None) + self.assertEqual(len(warned), 1) + self.assertIn('badkey', warned[0]) def test_dictionary_versions_key(self): from gcloud_bigtable.column_family import MaxVersionsGCRule From ab2b86a999dfeedf48c6a34ff4bac36ffcb3e199 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 14:15:42 -0800 Subject: [PATCH 56/65] Upstream changes. --- gcloud_bigtable/happybase/batch.py | 96 ++++----- gcloud_bigtable/happybase/table.py | 309 ++++++++++++++--------------- 2 files changed, 202 insertions(+), 203 deletions(-) diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index 4fb15b8..c067ea6 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -35,54 +35,6 @@ 'supported by Cloud Bigtable.') -def _get_column_pairs(columns, require_qualifier=False): - """Turns a list of column or column families in parsed pairs. - - Turns a column family (``fam`` or ``fam:``) into a pair such - as ``['fam', None]`` and turns a column (``fam:col``) into - ``['fam', 'col']``. - - :type columns: list - :param columns: Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type require_qualifier: bool - :param require_qualifier: Boolean indicating if the columns should - all have a qualifier or not. - - :rtype: list - :returns: List of pairs, where the first element in each pair is the - column family and the second is the column qualifier - (or :data:`None`). - :raises: :class:`ValueError ` if any of the columns - are not of the expected format. - :class:`ValueError ` if - ``require_qualifier`` is :data:`True` and one of the values is - for an entire column family - """ - column_pairs = [] - for column in columns: - # Remove trailing colons (i.e. for standalone column family). - column = column.rstrip(':') - num_colons = column.count(':') - if num_colons == 0: - # column is a column family. - if require_qualifier: - raise ValueError('column does not contain a qualifier', - column) - else: - column_pairs.append([column, None]) - elif num_colons == 1: - column_pairs.append(column.split(':')) - else: - raise ValueError('Column contains the : separator more than once') - - return column_pairs - - class Batch(object): """Batch class for accumulating mutations. @@ -322,3 +274,51 @@ def __exit__(self, exc_type, exc_value, traceback): # NOTE: For non-transactional batches, this will even commit mutations # if an error occurred during the context manager. self.send() + + +def _get_column_pairs(columns, require_qualifier=False): + """Turns a list of column or column families in parsed pairs. + + Turns a column family (``fam`` or ``fam:``) into a pair such + as ``['fam', None]`` and turns a column (``fam:col``) into + ``['fam', 'col']``. + + :type columns: list + :param columns: Iterable containing column names (as + strings). Each column name can be either + + * an entire column family: ``fam`` or ``fam:`` + * an single column: ``fam:col`` + + :type require_qualifier: bool + :param require_qualifier: Boolean indicating if the columns should + all have a qualifier or not. + + :rtype: list + :returns: List of pairs, where the first element in each pair is the + column family and the second is the column qualifier + (or :data:`None`). + :raises: :class:`ValueError ` if any of the columns + are not of the expected format. + :class:`ValueError ` if + ``require_qualifier`` is :data:`True` and one of the values is + for an entire column family + """ + column_pairs = [] + for column in columns: + # Remove trailing colons (i.e. for standalone column family). + column = column.rstrip(':') + num_colons = column.count(':') + if num_colons == 0: + # column is a column family. + if require_qualifier: + raise ValueError('column does not contain a qualifier', + column) + else: + column_pairs.append([column, None]) + elif num_colons == 1: + column_pairs.append(column.split(':')) + else: + raise ValueError('Column contains the : separator more than once') + + return column_pairs diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index dd6d8ea..50f06a6 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -40,10 +40,10 @@ _UNPACK_I64 = struct.Struct('>q').unpack -_DEFAULT_BATCH_SIZE = object() -_DEFAULT_SCAN_BATCHING = object() -_DEFAULT_SORTED_COLUMNS = object() _SIMPLE_GC_RULES = (MaxAgeGCRule, MaxVersionsGCRule) +# Upstream renames +_datetime_from_microseconds = _microseconds_to_timestamp +_microseconds_from_datetime = _timestamp_to_microseconds def make_row(cell_map, include_timestamp): @@ -94,54 +94,6 @@ def make_ordered_row(sorted_columns, include_timestamp): 'helper can not be implemented.', 'Called with', sorted_columns, include_timestamp) -def _convert_to_time_range(timestamp=None): - """Create a timestamp range from an HBase / HappyBase timestamp. - - HBase uses timestamp as an argument to specify an exclusive end - deadline. Cloud Bigtable also uses exclusive end times, so - the behavior matches. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). Intended to be used as the end of an HBase - time range, which is exclusive. - - :rtype: :class:`.TimestampRange`, :data:`NoneType ` - :returns: The timestamp range corresponding to the passed in - ``timestamp``. - """ - if timestamp is None: - return None - - next_timestamp = _microseconds_to_timestamp(1000 * timestamp) - return TimestampRange(end=next_timestamp) - - -def _cells_to_pairs(cells, include_timestamp=False): - """Converts list of cells to HappyBase format. - - :type cells: list - :param cells: List of :class:`.Cell` returned from a read request. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: List of values in the cell. If ``include_timestamp=True``, each - value will be a pair, with the first part the bytes value in - the cell and the second part the number of milliseconds in the - timestamp on the cell. - """ - result = [] - for cell in cells: - if include_timestamp: - ts_millis = _timestamp_to_microseconds(cell.timestamp) // 1000 - result.append((cell.value, ts_millis)) - else: - result.append(cell.value) - return result - def _filter_chain_helper(column=None, versions=None, timestamp=None, filters=None): @@ -250,90 +202,6 @@ def _row_keys_filter_helper(row_keys): return RowFilterUnion(filters=filters) -def _partial_row_to_dict(partial_row_data, include_timestamp=False): - """Convert a low-level row data object to a dictionary. - - Assumes only the latest value in each row are needed (e.g. different - behavior than in :meth:`Table.cells`). - - :type partial_row_data: :class:`.row_data.PartialRowData` - :param partial_row_data: Row data consumed from a stream. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: dict - :returns: The row data converted to a dictionary. - """ - result = {} - for column, cells in six.iteritems(partial_row_data.to_dict()): - cell_vals = _cells_to_pairs(cells, - include_timestamp=include_timestamp) - # NOTE: We assume there is exactly 1 version since we used that in - # our filter, but we don't check this. - result[column] = cell_vals[0] - return result - - -def _next_char(str_val, index): - """Gets the next character based on a position in a string. - - :type str_val: str - :param str_val: A string containing the character to update. - - :type index: int - :param index: An integer index in ``str_val``. - - :rtype: str - :returns: The next character after the character at ``index`` - in ``str_val``. - """ - if six.PY3: # pragma: NO COVER - ord_val = str_val[index] - else: - ord_val = ord(str_val[index]) - return _to_bytes(chr(ord_val + 1), encoding='latin-1') - - -def _string_successor(str_val): - """Increment and truncate a byte string. - - Determines shortest string that sorts after the given string when - compared using regular string comparison semantics. - - Borrowed from gcloud-golang. - - Increments the last byte that is smaller than ``0xFF``, and - drops everything after it. If the string only contains ``0xFF`` bytes, - ``''`` is returned. - - :type str_val: str - :param str_val: String to increment. - - :rtype: str - :returns: The next string in lexical order after ``str_val``. - """ - str_val = _to_bytes(str_val, encoding='latin-1') - if str_val == b'': - return str_val - - index = len(str_val) - 1 - while index >= 0: - if six.PY3: # pragma: NO COVER - if str_val[index] != 0xff: - break - else: - if str_val[index] != b'\xff': - break - index -= 1 - - if index == -1: - return b'' - - return str_val[:index] + _next_char(str_val, index) - - class Table(object): """Representation of Cloud Bigtable table. @@ -534,9 +402,7 @@ def cells(self, row, column, versions=None, timestamp=None, def scan(self, row_start=None, row_stop=None, row_prefix=None, columns=None, filter=None, timestamp=None, - include_timestamp=False, batch_size=_DEFAULT_BATCH_SIZE, - scan_batching=_DEFAULT_SCAN_BATCHING, - limit=None, sorted_columns=_DEFAULT_SORTED_COLUMNS): + include_timestamp=False, limit=None, **kwargs): """Create a scanner for data in this table. This method returns a generator that can be used for looping over the @@ -553,6 +419,17 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, omitted, a full table scan is done. Note that this usually results in severe performance problems. + The arguments ``batch_size``, ``scan_batching`` and ``sorted_columns`` + are allowed (as keyword arguments) for compatibility with + HappyBase. However, they will not be used in any way, and will cause a + warning if passed. (The ``batch_size`` determines the number of + results to retrieve per request. The HBase scanner defaults to reading + one record at a time, so this argument allows HappyBase to increase + that number. However, the Cloud Bigtable API uses HTTP/2 streaming so + there is no concept of a batch. The ``sorted_columns`` flag tells + HBase to return columns in order, but Cloud Bigtable doesn't have + this feature.) + :type row_start: str :param row_start: (Optional) Row key where the scanner should start (includes ``row_start``). If not specified, reads @@ -592,26 +469,12 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, :param include_timestamp: Flag to indicate if cell timestamps should be included with the output. - :type batch_size: int - :param batch_size: Unused parameter. This determines the number of - results to retrieve per request. The HBase scanner - defaults to reading one record at a time, so this - increases that number. The Cloud Bigtable API uses - HTTP/2 streaming and this value can't be set. - - :type scan_batching: bool - :param scan_batching: Unused parameter. Provided for compatibility - with HappyBase, but irrelevant for Cloud Bigtable - since it does not have concepts of batching or - caching for scans. - :type limit: int :param limit: (Optional) Maximum number of rows to return. - :type sorted_columns: bool - :param sorted_columns: Unused parameter. Provided compatibility with - HappyBase, but irrelevant for Cloud Bigtable - since it cannot return sorted columns. + :type kwargs: dict + :param kwargs: Remaining keyword arguments. Provided for HappyBase + compatibility. :raises: :class:`ValueError ` if ``batch_size`` or ``scan_batching`` are used, or if ``limit`` is set but @@ -763,6 +626,9 @@ def batch(self, timestamp=None, batch_size=None, transaction=False, Provided for compatibility with HappyBase, but irrelevant for Cloud Bigtable since it does not have a Write Ahead Log. + + :rtype: :class:`gcloud.bigtable.happybase.batch.Batch` + :returns: A batch bound to this table. """ return Batch(self, timestamp=timestamp, batch_size=batch_size, transaction=transaction, wal=wal) @@ -917,3 +783,136 @@ def _gc_rule_to_dict(gc_rule): if key1 != key2: result = {key1: rule1[key1], key2: rule2[key2]} return result + + +def _next_char(str_val, index): + """Gets the next character based on a position in a string. + + :type str_val: str + :param str_val: A string containing the character to update. + + :type index: int + :param index: An integer index in ``str_val``. + + :rtype: str + :returns: The next character after the character at ``index`` + in ``str_val``. + """ + if six.PY3: # pragma: NO COVER + ord_val = str_val[index] + else: + ord_val = ord(str_val[index]) + return _to_bytes(chr(ord_val + 1), encoding='latin-1') + + +def _string_successor(str_val): + """Increment and truncate a byte string. + + Determines shortest string that sorts after the given string when + compared using regular string comparison semantics. + + Modeled after implementation in ``gcloud-golang``. + + Increments the last byte that is smaller than ``0xFF``, and + drops everything after it. If the string only contains ``0xFF`` bytes, + ``''`` is returned. + + :type str_val: str + :param str_val: String to increment. + + :rtype: str + :returns: The next string in lexical order after ``str_val``. + """ + str_val = _to_bytes(str_val, encoding='latin-1') + if str_val == b'': + return str_val + + index = len(str_val) - 1 + while index >= 0: + if six.PY3: # pragma: NO COVER + if str_val[index] != 0xff: + break + else: + if str_val[index] != b'\xff': + break + index -= 1 + + if index == -1: + return b'' + + return str_val[:index] + _next_char(str_val, index) + + +def _convert_to_time_range(timestamp=None): + """Create a timestamp range from an HBase / HappyBase timestamp. + + HBase uses timestamp as an argument to specify an exclusive end + deadline. Cloud Bigtable also uses exclusive end times, so + the behavior matches. + + :type timestamp: int + :param timestamp: (Optional) Timestamp (in milliseconds since the + epoch). Intended to be used as the end of an HBase + time range, which is exclusive. + + :rtype: :class:`.TimestampRange`, :data:`NoneType ` + :returns: The timestamp range corresponding to the passed in + ``timestamp``. + """ + if timestamp is None: + return None + + next_timestamp = _datetime_from_microseconds(1000 * timestamp) + return TimestampRange(end=next_timestamp) + + +def _cells_to_pairs(cells, include_timestamp=False): + """Converts list of cells to HappyBase format. + + :type cells: list + :param cells: List of :class:`.Cell` returned from a read request. + + :type include_timestamp: bool + :param include_timestamp: Flag to indicate if cell timestamps should be + included with the output. + + :rtype: list + :returns: List of values in the cell. If ``include_timestamp=True``, each + value will be a pair, with the first part the bytes value in + the cell and the second part the number of milliseconds in the + timestamp on the cell. + """ + result = [] + for cell in cells: + if include_timestamp: + ts_millis = _microseconds_from_datetime(cell.timestamp) // 1000 + result.append((cell.value, ts_millis)) + else: + result.append(cell.value) + return result + + +def _partial_row_to_dict(partial_row_data, include_timestamp=False): + """Convert a low-level row data object to a dictionary. + + Assumes only the latest value in each row is needed (e.g. different + behavior than in :meth:`Table.cells`). + + :type partial_row_data: :class:`.row_data.PartialRowData` + :param partial_row_data: Row data consumed from a stream. + + :type include_timestamp: bool + :param include_timestamp: Flag to indicate if cell timestamps should be + included with the output. + + :rtype: dict + :returns: The row data converted to a dictionary. + """ + result = {} + for column, cells in six.iteritems(partial_row_data.to_dict()): + cell_vals = _cells_to_pairs(cells, + include_timestamp=include_timestamp) + # NOTE: We assume there is exactly 1 version since we used that in + # our filter, but we don't check this. + result[column] = cell_vals[0] + return result From 70af81e70a0ce5c6ff4ad267071cbfa47cf2f2b8 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 15:04:51 -0800 Subject: [PATCH 57/65] Upgrading to gRPC 0.13.0. --- gcloud_bigtable/__init__.py | 13 -- .../bigtable_cluster_service_pb2.py | 165 ---------------- .../_generated/bigtable_service_pb2.py | 145 -------------- .../_generated/bigtable_table_service_pb2.py | 185 ------------------ .../_generated/operations_grpc_pb2.py | 105 ---------- gcloud_bigtable/_helpers.py | 56 ++---- tox.ini | 2 +- 7 files changed, 15 insertions(+), 656 deletions(-) diff --git a/gcloud_bigtable/__init__.py b/gcloud_bigtable/__init__.py index f26e598..54bb2ec 100644 --- a/gcloud_bigtable/__init__.py +++ b/gcloud_bigtable/__init__.py @@ -15,17 +15,4 @@ """Google Cloud Bigtable API package.""" -from __future__ import print_function -import sys - -try: - from grpc._adapter import _c -except ImportError as exc: # pragma: NO COVER - if 'libgrpc.so' in str(exc): - print('gRPC libraries could not be located. Please see ' - 'instructions to locate these files. You\'ll want ' - 'to set your LD_LIBRARY_PATH variable to help ' - 'Python locate the libraries.', file=sys.stderr) - raise - from gcloud_bigtable.client import Client diff --git a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py index 7dab6ac..fa5e408 100644 --- a/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py @@ -35,173 +35,8 @@ DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n$com.google.bigtable.admin.cluster.v1B\034BigtableClusterServicesProtoP\001') import abc from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterBigtableClusterServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListClusters(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UndeleteCluster(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableClusterServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableClusterServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request): - raise NotImplementedError() - ListZones.async = None - @abc.abstractmethod - def GetCluster(self, request): - raise NotImplementedError() - GetCluster.async = None - @abc.abstractmethod - def ListClusters(self, request): - raise NotImplementedError() - ListClusters.async = None - @abc.abstractmethod - def CreateCluster(self, request): - raise NotImplementedError() - CreateCluster.async = None - @abc.abstractmethod - def UpdateCluster(self, request): - raise NotImplementedError() - UpdateCluster.async = None - @abc.abstractmethod - def DeleteCluster(self, request): - raise NotImplementedError() - DeleteCluster.async = None - @abc.abstractmethod - def UndeleteCluster(self, request): - raise NotImplementedError() - UndeleteCluster.async = None -def early_adopter_create_BigtableClusterService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.longrunning.operations_pb2 - method_service_descriptions = { - "CreateCluster": alpha_utilities.unary_unary_service_description( - servicer.CreateCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - "DeleteCluster": alpha_utilities.unary_unary_service_description( - servicer.DeleteCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "GetCluster": alpha_utilities.unary_unary_service_description( - servicer.GetCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - "ListClusters": alpha_utilities.unary_unary_service_description( - servicer.ListClusters, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, - ), - "ListZones": alpha_utilities.unary_unary_service_description( - servicer.ListZones, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, - ), - "UndeleteCluster": alpha_utilities.unary_unary_service_description( - servicer.UndeleteCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, - google.longrunning.operations_pb2.Operation.SerializeToString, - ), - "UpdateCluster": alpha_utilities.unary_unary_service_description( - servicer.UpdateCluster, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - } - return early_adopter_implementations.server("google.bigtable.admin.cluster.v1.BigtableClusterService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.longrunning.operations_pb2 - method_invocation_descriptions = { - "CreateCluster": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - "DeleteCluster": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "GetCluster": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - "ListClusters": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, - ), - "ListZones": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, - ), - "UndeleteCluster": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, - google.longrunning.operations_pb2.Operation.FromString, - ), - "UpdateCluster": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - } - return early_adopter_implementations.stub("google.bigtable.admin.cluster.v1.BigtableClusterService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaBigtableClusterServiceServicer(object): """""" diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py index 189b3a1..e2fdb8a 100644 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_service_pb2.py @@ -34,153 +34,8 @@ DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\025BigtableServicesProtoP\001\210\001\001') import abc from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterBigtableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def SampleRowKeys(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def MutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def MutateRows(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CheckAndMutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ReadModifyWriteRow(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request): - raise NotImplementedError() - ReadRows.async = None - @abc.abstractmethod - def SampleRowKeys(self, request): - raise NotImplementedError() - SampleRowKeys.async = None - @abc.abstractmethod - def MutateRow(self, request): - raise NotImplementedError() - MutateRow.async = None - @abc.abstractmethod - def MutateRows(self, request): - raise NotImplementedError() - MutateRows.async = None - @abc.abstractmethod - def CheckAndMutateRow(self, request): - raise NotImplementedError() - CheckAndMutateRow.async = None - @abc.abstractmethod - def ReadModifyWriteRow(self, request): - raise NotImplementedError() - ReadModifyWriteRow.async = None -def early_adopter_create_BigtableService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - method_service_descriptions = { - "CheckAndMutateRow": alpha_utilities.unary_unary_service_description( - servicer.CheckAndMutateRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, - ), - "MutateRow": alpha_utilities.unary_unary_service_description( - servicer.MutateRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "MutateRows": alpha_utilities.unary_unary_service_description( - servicer.MutateRows, - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.SerializeToString, - ), - "ReadModifyWriteRow": alpha_utilities.unary_unary_service_description( - servicer.ReadModifyWriteRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, - gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, - ), - "ReadRows": alpha_utilities.unary_stream_service_description( - servicer.ReadRows, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, - ), - "SampleRowKeys": alpha_utilities.unary_stream_service_description( - servicer.SampleRowKeys, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, - ), - } - return early_adopter_implementations.server("google.bigtable.v1.BigtableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - method_invocation_descriptions = { - "CheckAndMutateRow": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, - ), - "MutateRow": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "MutateRows": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.FromString, - ), - "ReadModifyWriteRow": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, - ), - "ReadRows": alpha_utilities.unary_stream_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, - ), - "SampleRowKeys": alpha_utilities.unary_stream_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, - ), - } - return early_adopter_implementations.stub("google.bigtable.v1.BigtableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaBigtableServiceServicer(object): """""" diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py index 5a3c977..4c27255 100644 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py @@ -34,193 +34,8 @@ DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\032BigtableTableServicesProtoP\001') import abc from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterBigtableTableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListTables(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def RenameTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteColumnFamily(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableTableServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableTableServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request): - raise NotImplementedError() - CreateTable.async = None - @abc.abstractmethod - def ListTables(self, request): - raise NotImplementedError() - ListTables.async = None - @abc.abstractmethod - def GetTable(self, request): - raise NotImplementedError() - GetTable.async = None - @abc.abstractmethod - def DeleteTable(self, request): - raise NotImplementedError() - DeleteTable.async = None - @abc.abstractmethod - def RenameTable(self, request): - raise NotImplementedError() - RenameTable.async = None - @abc.abstractmethod - def CreateColumnFamily(self, request): - raise NotImplementedError() - CreateColumnFamily.async = None - @abc.abstractmethod - def UpdateColumnFamily(self, request): - raise NotImplementedError() - UpdateColumnFamily.async = None - @abc.abstractmethod - def DeleteColumnFamily(self, request): - raise NotImplementedError() - DeleteColumnFamily.async = None -def early_adopter_create_BigtableTableService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - method_service_descriptions = { - "CreateColumnFamily": alpha_utilities.unary_unary_service_description( - servicer.CreateColumnFamily, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ), - "CreateTable": alpha_utilities.unary_unary_service_description( - servicer.CreateTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ), - "DeleteColumnFamily": alpha_utilities.unary_unary_service_description( - servicer.DeleteColumnFamily, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "DeleteTable": alpha_utilities.unary_unary_service_description( - servicer.DeleteTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "GetTable": alpha_utilities.unary_unary_service_description( - servicer.GetTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ), - "ListTables": alpha_utilities.unary_unary_service_description( - servicer.ListTables, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, - ), - "RenameTable": alpha_utilities.unary_unary_service_description( - servicer.RenameTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "UpdateColumnFamily": alpha_utilities.unary_unary_service_description( - servicer.UpdateColumnFamily, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ), - } - return early_adopter_implementations.server("google.bigtable.admin.table.v1.BigtableTableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableTableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - method_invocation_descriptions = { - "CreateColumnFamily": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - ), - "CreateTable": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ), - "DeleteColumnFamily": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "DeleteTable": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "GetTable": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ), - "ListTables": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, - ), - "RenameTable": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "UpdateColumnFamily": alpha_utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - ), - } - return early_adopter_implementations.stub("google.bigtable.admin.table.v1.BigtableTableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaBigtableTableServiceServicer(object): """""" diff --git a/gcloud_bigtable/_generated/operations_grpc_pb2.py b/gcloud_bigtable/_generated/operations_grpc_pb2.py index 998d244..e4911b3 100644 --- a/gcloud_bigtable/_generated/operations_grpc_pb2.py +++ b/gcloud_bigtable/_generated/operations_grpc_pb2.py @@ -1,112 +1,7 @@ import abc from grpc.beta import implementations as beta_implementations -from grpc.early_adopter import implementations as early_adopter_implementations -from grpc.framework.alpha import utilities as alpha_utilities from grpc.framework.common import cardinality from grpc.framework.interfaces.face import utilities as face_utilities -class EarlyAdopterOperationsServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListOperations(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CancelOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteOperation(self, request, context): - raise NotImplementedError() -class EarlyAdopterOperationsServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterOperationsStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request): - raise NotImplementedError() - GetOperation.async = None - @abc.abstractmethod - def ListOperations(self, request): - raise NotImplementedError() - ListOperations.async = None - @abc.abstractmethod - def CancelOperation(self, request): - raise NotImplementedError() - CancelOperation.async = None - @abc.abstractmethod - def DeleteOperation(self, request): - raise NotImplementedError() - DeleteOperation.async = None -def early_adopter_create_Operations_server(servicer, port, private_key=None, certificate_chain=None): - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - method_service_descriptions = { - "CancelOperation": alpha_utilities.unary_unary_service_description( - servicer.CancelOperation, - google.longrunning.operations_pb2.CancelOperationRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "DeleteOperation": alpha_utilities.unary_unary_service_description( - servicer.DeleteOperation, - google.longrunning.operations_pb2.DeleteOperationRequest.FromString, - google.protobuf.empty_pb2.Empty.SerializeToString, - ), - "GetOperation": alpha_utilities.unary_unary_service_description( - servicer.GetOperation, - google.longrunning.operations_pb2.GetOperationRequest.FromString, - google.longrunning.operations_pb2.Operation.SerializeToString, - ), - "ListOperations": alpha_utilities.unary_unary_service_description( - servicer.ListOperations, - google.longrunning.operations_pb2.ListOperationsRequest.FromString, - google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, - ), - } - return early_adopter_implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - method_invocation_descriptions = { - "CancelOperation": alpha_utilities.unary_unary_invocation_description( - google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "DeleteOperation": alpha_utilities.unary_unary_invocation_description( - google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, - google.protobuf.empty_pb2.Empty.FromString, - ), - "GetOperation": alpha_utilities.unary_unary_invocation_description( - google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, - google.longrunning.operations_pb2.Operation.FromString, - ), - "ListOperations": alpha_utilities.unary_unary_invocation_description( - google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, - google.longrunning.operations_pb2.ListOperationsResponse.FromString, - ), - } - return early_adopter_implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) class BetaOperationsServicer(object): """""" diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py index 45c1a30..55f9303 100644 --- a/gcloud_bigtable/_helpers.py +++ b/gcloud_bigtable/_helpers.py @@ -19,22 +19,10 @@ """ -import platform - from grpc.beta import implementations -# See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 -# for appropriate values on other systems. -_PLAT_SYS = platform.system() -SSL_CERT_FILE = None -if _PLAT_SYS == 'Linux': - SSL_CERT_FILE = '/etc/ssl/certs/ca-certificates.crt' -elif _PLAT_SYS == 'Darwin': # pragma: NO COVER - SSL_CERT_FILE = '/usr/local/etc/openssl/cert.pem' - - -class MetadataTransformer(object): +class MetadataPlugin(object): """Callable class to transform metadata for gRPC requests. :type client: :class:`.client.Client` @@ -46,32 +34,14 @@ def __init__(self, client): self._credentials = client.credentials self._user_agent = client.user_agent - def __call__(self, ignored_val): + def __call__(self, unused_context, callback): """Adds authorization header to request metadata.""" access_token = self._credentials.get_access_token().access_token - return [ + headers = [ ('Authorization', 'Bearer ' + access_token), ('User-agent', self._user_agent), ] - - -def get_certs(): - """Gets the root certificates. - - .. note:: - - This is only called by :func:`make_stub`. For most applications, - a few gRPC stubs (four total, one for each service) will be created - when a :class:`.Client` is created. This function will not likely - be used again while that application is running. - - However, it may be worthwhile to cache the output of this function. - - :rtype: str - :returns: The root certificates for the current machine. - """ - with open(SSL_CERT_FILE, mode='rb') as file_obj: - return file_obj.read() + callback(headers, None) def make_stub(client, stub_factory, host, port): @@ -96,11 +66,13 @@ def make_stub(client, stub_factory, host, port): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: The stub object used to make gRPC requests to a given API. """ - root_certificates = get_certs() - client_credentials = implementations.ssl_client_credentials( - root_certificates, private_key=None, certificate_chain=None) - channel = implementations.secure_channel( - host, port, client_credentials) - custom_metadata_transformer = MetadataTransformer(client) - return stub_factory(channel, - metadata_transformer=custom_metadata_transformer) + # Leaving the first argument to ssl_channel_credentials() as None + # loads root certificates from `grpc/_adapter/credentials/roots.pem`. + transport_creds = implementations.ssl_channel_credentials(None, None, None) + custom_metadata_plugin = MetadataPlugin(client) + auth_creds = implementations.metadata_call_credentials( + custom_metadata_plugin, name='google_creds') + channel_creds = implementations.composite_channel_credentials( + transport_creds, auth_creds) + channel = implementations.secure_channel(host, port, channel_creds) + return stub_factory(channel) diff --git a/tox.ini b/tox.ini index c83cd1b..db7da7e 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ commands = nosetests -v {toxinidir}/system_tests/run.py nosetests -v {toxinidir}/system_tests/run_happybase.py deps = - grpcio==0.11.0b1 + grpcio==0.13.0 nose unittest2 passenv = GOOGLE_* GCLOUD_* GRPC_TRACE From ac1726272e8417efe77b1d4737d54874351e700e Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 15:41:30 -0800 Subject: [PATCH 58/65] Fixing some broken tests (caused by haphazard changes.) --- gcloud_bigtable/happybase/batch.py | 2 + gcloud_bigtable/happybase/table.py | 31 +++++---- gcloud_bigtable/happybase/test_table.py | 71 ++++++++++++++++++-- gcloud_bigtable/test__helpers.py | 88 ++++++++++--------------- 4 files changed, 122 insertions(+), 70 deletions(-) diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index c067ea6..0bbf7f0 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -25,7 +25,9 @@ # BEGIN: Renames for upstream. +# pylint: disable=invalid-name _datetime_from_microseconds = _microseconds_to_timestamp +# pylint: enable=invalid-name # END: Renames for upstream. _WAL_SENTINEL = object() # Assumed granularity of timestamps in Cloud Bigtable. diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index 50f06a6..48a2cfb 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -16,6 +16,7 @@ import struct +import warnings import six @@ -39,11 +40,14 @@ from gcloud_bigtable.table import Table as _LowLevelTable +_WARN = warnings.warn _UNPACK_I64 = struct.Struct('>q').unpack _SIMPLE_GC_RULES = (MaxAgeGCRule, MaxVersionsGCRule) # Upstream renames +# pylint: disable=invalid-name _datetime_from_microseconds = _microseconds_to_timestamp _microseconds_from_datetime = _timestamp_to_microseconds +# pylint: enable=invalid-name def make_row(cell_map, include_timestamp): @@ -476,21 +480,24 @@ def scan(self, row_start=None, row_stop=None, row_prefix=None, :param kwargs: Remaining keyword arguments. Provided for HappyBase compatibility. - :raises: :class:`ValueError ` if ``batch_size`` - or ``scan_batching`` are used, or if ``limit`` is set but - non-positive, or if row prefix is used with row start/stop, + :raises: If ``limit`` is set but non-positive, or if row prefix is + used with row start/stop, :class:`TypeError ` if a string ``filter`` is used. """ - if batch_size is not _DEFAULT_BATCH_SIZE: - raise ValueError('Batch size cannot be set for gcloud ' - 'HappyBase module') - if scan_batching is not _DEFAULT_SCAN_BATCHING: - raise ValueError('Scan batching cannot be set for gcloud ' - 'HappyBase module') - if sorted_columns is not _DEFAULT_SORTED_COLUMNS: - raise ValueError('Sorted columns cannot be set for gcloud ' - 'HappyBase module') + legacy_args = [] + for kw_name in ('batch_size', 'scan_batching', 'sorted_columns'): + if kw_name in kwargs: + legacy_args.append(kw_name) + kwargs.pop(kw_name) + if legacy_args: + legacy_args = ', '.join(legacy_args) + message = ('The HappyBase legacy arguments %s were used. These ' + 'arguments are unused by gcloud.' % (legacy_args,)) + _WARN(message) + if kwargs: + raise TypeError('Received unexpected arguments', kwargs.keys()) + if limit is not None and limit < 1: raise ValueError('limit must be positive') if row_prefix is not None: diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py index 3db5fc2..1b8d278 100644 --- a/gcloud_bigtable/happybase/test_table.py +++ b/gcloud_bigtable/happybase/test_table.py @@ -806,25 +806,67 @@ def test_cells_with_results(self): self, [(fake_cells,)], [to_pairs_kwargs]) def test_scan_with_batch_size(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import table as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + name = 'table-name' connection = None table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(batch_size=object())) + # Use unknown to force a TypeError, so we don't need to + # stub out the rest of the method. + with self.assertRaises(TypeError): + with _Monkey(MUT, _WARN=mock_warn): + list(table.scan(batch_size=object(), unknown=None)) + + self.assertEqual(len(warned), 1) + self.assertIn('batch_size', warned[0]) def test_scan_with_scan_batching(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import table as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + name = 'table-name' connection = None table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(scan_batching=object())) + # Use unknown to force a TypeError, so we don't need to + # stub out the rest of the method. + with self.assertRaises(TypeError): + with _Monkey(MUT, _WARN=mock_warn): + list(table.scan(scan_batching=object(), unknown=None)) + + self.assertEqual(len(warned), 1) + self.assertIn('scan_batching', warned[0]) def test_scan_with_sorted_columns(self): + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import table as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + name = 'table-name' connection = None table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(sorted_columns=object())) + # Use unknown to force a TypeError, so we don't need to + # stub out the rest of the method. + with self.assertRaises(TypeError): + with _Monkey(MUT, _WARN=mock_warn): + list(table.scan(sorted_columns=object(), unknown=None)) + + self.assertEqual(len(warned), 1) + self.assertIn('sorted_columns', warned[0]) def test_scan_with_invalid_limit(self): name = 'table-name' @@ -1155,6 +1197,23 @@ def test_counter_inc(self): } self._counter_inc_helper(row, column, value, commit_result) + def test_counter_inc_unicode(self): + import struct + + row = 'row-key' + col_fam = b'fam' + col_qual = b'col1' + column = (col_fam + b':' + col_qual).decode('utf-8') + value = 42 + packed_value = struct.pack('>q', value) + fake_timestamp = None + commit_result = { + col_fam: { + col_qual: [(packed_value, fake_timestamp)], + } + } + self._counter_inc_helper(row, column, value, commit_result) + def test_counter_inc_bad_result(self): row = 'row-key' col_fam = 'fam' diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py index 7d3b1c5..36c45df 100644 --- a/gcloud_bigtable/test__helpers.py +++ b/gcloud_bigtable/test__helpers.py @@ -16,11 +16,11 @@ import unittest2 -class TestMetadataTransformer(unittest2.TestCase): +class TestMetadataPlugin(unittest2.TestCase): def _getTargetClass(self): - from gcloud_bigtable._helpers import MetadataTransformer - return MetadataTransformer + from gcloud_bigtable._helpers import MetadataPlugin + return MetadataPlugin def _makeOne(self, *args, **kwargs): return self._getTargetClass()(*args, **kwargs) @@ -36,9 +36,9 @@ def test_constructor(self): user_agent = 'USER_AGENT' client = Client(project=project, credentials=credentials, user_agent=user_agent) - transformer = self._makeOne(client) - self.assertTrue(transformer._credentials is scoped_creds) - self.assertEqual(transformer._user_agent, user_agent) + plugin = self._makeOne(client) + self.assertTrue(plugin._credentials is scoped_creds) + self.assertEqual(plugin._user_agent, user_agent) self.assertEqual(credentials._called, [ ('create_scoped', ([DATA_SCOPE],), {}), ]) @@ -54,48 +54,30 @@ def test___call__(self): class _ReturnVal(object): access_token = access_token_expected + callback_args = [] + + def callback(value1, value2): + callback_args.append((value1, value2)) + scoped_creds = _MockWithAttachedMethods(_ReturnVal) credentials = _MockWithAttachedMethods(scoped_creds) project = 'PROJECT' client = Client(project=project, credentials=credentials) - transformer = self._makeOne(client) - result = transformer(None) - self.assertEqual( - result, - [ - ('Authorization', 'Bearer ' + access_token_expected), - ('User-agent', DEFAULT_USER_AGENT), - ]) + plugin = self._makeOne(client) + result = plugin(None, callback) + self.assertEqual(result, None) + cb_headers = [ + ('Authorization', 'Bearer ' + access_token_expected), + ('User-agent', DEFAULT_USER_AGENT), + ] + self.assertEqual(callback_args, [(cb_headers, None)]) self.assertEqual(credentials._called, [ ('create_scoped', ([DATA_SCOPE],), {}), ]) self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) -class Test_get_certs(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._helpers import get_certs - return get_certs() - - def test_it(self): - import tempfile - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - # Just write to a mock file. - filename = tempfile.mktemp() - contents = b'FOOBARBAZ' - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - with _Monkey(MUT, SSL_CERT_FILE=filename): - result = self._callFUT() - - self.assertEqual(result, contents) - - class Test_make_stub(unittest2.TestCase): def _callFUT(self, credentials, stub_factory, host, port): @@ -110,35 +92,37 @@ def test_it(self): mock_result = object() custom_factory = _MockCalled(mock_result) - transformed = object() - transformer = _MockCalled(transformed) + mock_plugin = object() + plugin = _MockCalled(mock_plugin) - client_creds = object() + ssl_creds = object() + metadata_creds = object() + composite_creds = object() channel = object() - implementations_mod = _MockWithAttachedMethods(client_creds, channel) + implementations_mod = _MockWithAttachedMethods( + ssl_creds, metadata_creds, composite_creds, channel) host = 'HOST' port = 1025 - certs = 'FOOBAR' client = _MockWithAttachedMethods() - with _Monkey(MUT, get_certs=lambda: certs, - implementations=implementations_mod, - MetadataTransformer=transformer): + with _Monkey(MUT, implementations=implementations_mod, + MetadataPlugin=plugin): result = self._callFUT(client, custom_factory, host, port) self.assertTrue(result is mock_result) custom_factory.check_called( self, [(channel,)], - [{ - 'metadata_transformer': transformed, - }], + [{}], ) - transformer.check_called(self, [(client,)]) + plugin.check_called(self, [(client,)]) self.assertEqual(client._called, []) # Check what was called on the module. self.assertEqual(implementations_mod._called, [ - ('ssl_client_credentials', (certs,), - {'private_key': None, 'certificate_chain': None}), - ('secure_channel', (host, port, client_creds), {}), + ('ssl_channel_credentials', (None, None, None), {}), + ('metadata_call_credentials', (mock_plugin,), + {'name': 'google_creds'}), + ('composite_channel_credentials', + (ssl_creds, metadata_creds), {}), + ('secure_channel', (host, port, composite_creds), {}), ]) From cf646c59417f01b91a6dfb84f6b47066f941ac83 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 24 Feb 2016 16:24:14 -0800 Subject: [PATCH 59/65] Removing _helpers module. --- gcloud_bigtable/_helpers.py | 78 ------------------- gcloud_bigtable/client.py | 75 +++++++++++++++--- gcloud_bigtable/test__helpers.py | 128 ------------------------------- gcloud_bigtable/test_client.py | 122 +++++++++++++++++++++++++++-- 4 files changed, 183 insertions(+), 220 deletions(-) delete mode 100644 gcloud_bigtable/_helpers.py delete mode 100644 gcloud_bigtable/test__helpers.py diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py deleted file mode 100644 index 55f9303..0000000 --- a/gcloud_bigtable/_helpers.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility methods for gcloud Bigtable. - -Primarily includes helpers for dealing with low-level -protobuf objects. -""" - - -from grpc.beta import implementations - - -class MetadataPlugin(object): - """Callable class to transform metadata for gRPC requests. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - """ - - def __init__(self, client): - self._credentials = client.credentials - self._user_agent = client.user_agent - - def __call__(self, unused_context, callback): - """Adds authorization header to request metadata.""" - access_token = self._credentials.get_access_token().access_token - headers = [ - ('Authorization', 'Bearer ' + access_token), - ('User-agent', self._user_agent), - ] - callback(headers, None) - - -def make_stub(client, stub_factory, host, port): - """Makes a stub for an RPC service. - - Uses / depends on the beta implementation of gRPC. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - - :type stub_factory: callable - :param stub_factory: A factory which will create a gRPC stub for - a given service. - - :type host: str - :param host: The host for the service. - - :type port: int - :param port: The port for the service. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: The stub object used to make gRPC requests to a given API. - """ - # Leaving the first argument to ssl_channel_credentials() as None - # loads root certificates from `grpc/_adapter/credentials/roots.pem`. - transport_creds = implementations.ssl_channel_credentials(None, None, None) - custom_metadata_plugin = MetadataPlugin(client) - auth_creds = implementations.metadata_call_credentials( - custom_metadata_plugin, name='google_creds') - channel_creds = implementations.composite_channel_credentials( - transport_creds, auth_creds) - channel = implementations.secure_channel(host, port, channel_creds) - return stub_factory(channel) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py index 271e712..7e12455 100644 --- a/gcloud_bigtable/client.py +++ b/gcloud_bigtable/client.py @@ -29,6 +29,8 @@ import copy +from grpc.beta import implementations + from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 from gcloud_bigtable._generated import ( bigtable_cluster_service_messages_pb2 as messages_pb2) @@ -36,7 +38,6 @@ from gcloud_bigtable._generated import bigtable_service_pb2 from gcloud_bigtable._generated import bigtable_table_service_pb2 from gcloud_bigtable._generated import operations_grpc_pb2 -from gcloud_bigtable._helpers import make_stub from gcloud_bigtable._non_upstream_helpers import _FactoryMixin from gcloud_bigtable._non_upstream_helpers import get_credentials from gcloud_bigtable.cluster import Cluster @@ -268,8 +269,8 @@ def _make_data_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, DATA_STUB_FACTORY, - DATA_API_HOST, DATA_API_PORT) + return _make_stub(self, DATA_STUB_FACTORY, + DATA_API_HOST, DATA_API_PORT) def _make_cluster_stub(self): """Creates gRPC stub to make requests to the Cluster Admin API. @@ -277,8 +278,8 @@ def _make_cluster_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return _make_stub(self, CLUSTER_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_operations_stub(self): """Creates gRPC stub to make requests to the Operations API. @@ -289,8 +290,8 @@ def _make_operations_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) + return _make_stub(self, OPERATIONS_STUB_FACTORY, + CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) def _make_table_stub(self): """Creates gRPC stub to make requests to the Table Admin API. @@ -298,8 +299,8 @@ def _make_table_stub(self): :rtype: :class:`grpc.beta._stub._AutoIntermediary` :returns: A gRPC stub object. """ - return make_stub(self, TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) + return _make_stub(self, TABLE_STUB_FACTORY, + TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) def is_started(self): """Check if the client has been started. @@ -416,3 +417,59 @@ def list_clusters(self): clusters = [Cluster.from_pb(cluster_pb, self) for cluster_pb in list_clusters_response.clusters] return clusters, failed_zones + + +class _MetadataPlugin(object): + """Callable class to transform metadata for gRPC requests. + + :type client: :class:`.client.Client` + :param client: The client that owns the cluster. Provides authorization and + user agent. + """ + + def __init__(self, client): + self._credentials = client.credentials + self._user_agent = client.user_agent + + def __call__(self, unused_context, callback): + """Adds authorization header to request metadata.""" + access_token = self._credentials.get_access_token().access_token + headers = [ + ('Authorization', 'Bearer ' + access_token), + ('User-agent', self._user_agent), + ] + callback(headers, None) + + +def _make_stub(client, stub_factory, host, port): + """Makes a stub for an RPC service. + + Uses / depends on the beta implementation of gRPC. + + :type client: :class:`.client.Client` + :param client: The client that owns the cluster. Provides authorization and + user agent. + + :type stub_factory: callable + :param stub_factory: A factory which will create a gRPC stub for + a given service. + + :type host: str + :param host: The host for the service. + + :type port: int + :param port: The port for the service. + + :rtype: :class:`grpc.beta._stub._AutoIntermediary` + :returns: The stub object used to make gRPC requests to a given API. + """ + # Leaving the first argument to ssl_channel_credentials() as None + # loads root certificates from `grpc/_adapter/credentials/roots.pem`. + transport_creds = implementations.ssl_channel_credentials(None, None, None) + custom_metadata_plugin = _MetadataPlugin(client) + auth_creds = implementations.metadata_call_credentials( + custom_metadata_plugin, name='google_creds') + channel_creds = implementations.composite_channel_credentials( + transport_creds, auth_creds) + channel = implementations.secure_channel(host, port, channel_creds) + return stub_factory(channel) diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py deleted file mode 100644 index 36c45df..0000000 --- a/gcloud_bigtable/test__helpers.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestMetadataPlugin(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable._helpers import MetadataPlugin - return MetadataPlugin - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - user_agent = 'USER_AGENT' - client = Client(project=project, credentials=credentials, - user_agent=user_agent) - plugin = self._makeOne(client) - self.assertTrue(plugin._credentials is scoped_creds) - self.assertEqual(plugin._user_agent, user_agent) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - - def test___call__(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - from gcloud_bigtable.client import DEFAULT_USER_AGENT - - access_token_expected = 'FOOBARBAZ' - - class _ReturnVal(object): - access_token = access_token_expected - - callback_args = [] - - def callback(value1, value2): - callback_args.append((value1, value2)) - - scoped_creds = _MockWithAttachedMethods(_ReturnVal) - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - client = Client(project=project, credentials=credentials) - - plugin = self._makeOne(client) - result = plugin(None, callback) - self.assertEqual(result, None) - cb_headers = [ - ('Authorization', 'Bearer ' + access_token_expected), - ('User-agent', DEFAULT_USER_AGENT), - ] - self.assertEqual(callback_args, [(cb_headers, None)]) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) - - -class Test_make_stub(unittest2.TestCase): - - def _callFUT(self, credentials, stub_factory, host, port): - from gcloud_bigtable._helpers import make_stub - return make_stub(credentials, stub_factory, host, port) - - def test_it(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - mock_result = object() - custom_factory = _MockCalled(mock_result) - mock_plugin = object() - plugin = _MockCalled(mock_plugin) - - ssl_creds = object() - metadata_creds = object() - composite_creds = object() - channel = object() - implementations_mod = _MockWithAttachedMethods( - ssl_creds, metadata_creds, composite_creds, channel) - - host = 'HOST' - port = 1025 - client = _MockWithAttachedMethods() - with _Monkey(MUT, implementations=implementations_mod, - MetadataPlugin=plugin): - result = self._callFUT(client, custom_factory, host, port) - - self.assertTrue(result is mock_result) - custom_factory.check_called( - self, - [(channel,)], - [{}], - ) - plugin.check_called(self, [(client,)]) - self.assertEqual(client._called, []) - # Check what was called on the module. - self.assertEqual(implementations_mod._called, [ - ('ssl_channel_credentials', (None, None, None), {}), - ('metadata_call_credentials', (mock_plugin,), - {'name': 'google_creds'}), - ('composite_channel_credentials', - (ssl_creds, metadata_creds), {}), - ('secure_channel', (host, port, composite_creds), {}), - ]) diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py index cbf0579..c1bbf30 100644 --- a/gcloud_bigtable/test_client.py +++ b/gcloud_bigtable/test_client.py @@ -291,7 +291,7 @@ def test__make_data_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, _make_stub=mock_make_stub): result = client._make_data_stub() self.assertTrue(result is expected_result) @@ -319,7 +319,7 @@ def test__make_cluster_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, _make_stub=mock_make_stub): result = client._make_cluster_stub() self.assertTrue(result is expected_result) @@ -347,7 +347,7 @@ def test__make_operations_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, _make_stub=mock_make_stub): result = client._make_operations_stub() self.assertTrue(result is expected_result) @@ -375,7 +375,7 @@ def test__make_table_stub(self): client = self._makeOne(project=PROJECT, credentials=credentials) expected_result = object() mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, _make_stub=mock_make_stub): result = client._make_table_stub() self.assertTrue(result is expected_result) @@ -413,7 +413,7 @@ def _start_method_helper(self, admin): credentials=credentials, admin=admin) stub = _FakeStub() mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, make_stub=mock_make_stub): + with _Monkey(MUT, _make_stub=mock_make_stub): client.start() self.assertTrue(client._data_stub_internal is stub) @@ -637,6 +637,118 @@ def test_list_clusters(self): )]) +class Test_MetadataPlugin(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud_bigtable.client import _MetadataPlugin + return _MetadataPlugin + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def test_constructor(self): + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable.client import Client + from gcloud_bigtable.client import DATA_SCOPE + + scoped_creds = object() + credentials = _MockWithAttachedMethods(scoped_creds) + project = 'PROJECT' + user_agent = 'USER_AGENT' + client = Client(project=project, credentials=credentials, + user_agent=user_agent) + plugin = self._makeOne(client) + self.assertTrue(plugin._credentials is scoped_creds) + self.assertEqual(plugin._user_agent, user_agent) + self.assertEqual(credentials._called, [ + ('create_scoped', ([DATA_SCOPE],), {}), + ]) + + def test___call__(self): + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable.client import Client + from gcloud_bigtable.client import DATA_SCOPE + from gcloud_bigtable.client import DEFAULT_USER_AGENT + + access_token_expected = 'FOOBARBAZ' + + class _ReturnVal(object): + access_token = access_token_expected + + callback_args = [] + + def callback(value1, value2): + callback_args.append((value1, value2)) + + scoped_creds = _MockWithAttachedMethods(_ReturnVal) + credentials = _MockWithAttachedMethods(scoped_creds) + project = 'PROJECT' + client = Client(project=project, credentials=credentials) + + plugin = self._makeOne(client) + result = plugin(None, callback) + self.assertEqual(result, None) + cb_headers = [ + ('Authorization', 'Bearer ' + access_token_expected), + ('User-agent', DEFAULT_USER_AGENT), + ] + self.assertEqual(callback_args, [(cb_headers, None)]) + self.assertEqual(credentials._called, [ + ('create_scoped', ([DATA_SCOPE],), {}), + ]) + self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) + + +class Test__make_stub(unittest2.TestCase): + + def _callFUT(self, credentials, stub_factory, host, port): + from gcloud_bigtable.client import _make_stub + return _make_stub(credentials, stub_factory, host, port) + + def test_it(self): + from gcloud_bigtable._testing import _MockCalled + from gcloud_bigtable._testing import _MockWithAttachedMethods + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable import client as MUT + + mock_result = object() + custom_factory = _MockCalled(mock_result) + mock_plugin = object() + plugin = _MockCalled(mock_plugin) + + ssl_creds = object() + metadata_creds = object() + composite_creds = object() + channel = object() + implementations_mod = _MockWithAttachedMethods( + ssl_creds, metadata_creds, composite_creds, channel) + + host = 'HOST' + port = 1025 + client = _MockWithAttachedMethods() + with _Monkey(MUT, implementations=implementations_mod, + _MetadataPlugin=plugin): + result = self._callFUT(client, custom_factory, host, port) + + self.assertTrue(result is mock_result) + custom_factory.check_called( + self, + [(channel,)], + [{}], + ) + plugin.check_called(self, [(client,)]) + self.assertEqual(client._called, []) + # Check what was called on the module. + self.assertEqual(implementations_mod._called, [ + ('ssl_channel_credentials', (None, None, None), {}), + ('metadata_call_credentials', (mock_plugin,), + {'name': 'google_creds'}), + ('composite_channel_credentials', + (ssl_creds, metadata_creds), {}), + ('secure_channel', (host, port, composite_creds), {}), + ]) + + class _FakeStub(object): def __init__(self): From 007cb866b957697bf8c355cd9cb672aa64e74a74 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sun, 28 Feb 2016 15:58:38 -0800 Subject: [PATCH 60/65] Adding changes to batch. --- gcloud_bigtable/happybase/batch.py | 2 +- system_tests/run.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index 0bbf7f0..78fc516 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -279,7 +279,7 @@ def __exit__(self, exc_type, exc_value, traceback): def _get_column_pairs(columns, require_qualifier=False): - """Turns a list of column or column families in parsed pairs. + """Turns a list of column or column families into parsed pairs. Turns a column family (``fam`` or ``fam:``) into a pair such as ``['fam', None]`` and turns a column (``fam:col``) into diff --git a/system_tests/run.py b/system_tests/run.py index f95f164..c4e7d2e 100644 --- a/system_tests/run.py +++ b/system_tests/run.py @@ -40,7 +40,6 @@ CENTRAL_1C_ZONE = 'us-central1-c' NOW_MILLIS = int(1000 * time.time()) CLUSTER_ID = 'gcloud-python-%d' % (NOW_MILLIS,) -SERVE_NODES = 3 TABLE_ID = 'gcloud-python-test-table' COLUMN_FAMILY_ID1 = u'col-fam-id1' COLUMN_FAMILY_ID2 = u'col-fam-id2' @@ -53,13 +52,13 @@ CELL_VAL4 = b'foo' ROW_KEY = b'row-key' ROW_KEY_ALT = b'row-key-alt' +EXISTING_CLUSTERS = [] EXPECTED_ZONES = ( 'asia-east1-b', 'europe-west1-c', 'us-central1-b', CENTRAL_1C_ZONE, ) -EXISTING_CLUSTERS = [] CREDENTIALS = GoogleCredentials.get_application_default() CLIENT = Client(project=PROJECT_ID, credentials=CREDENTIALS, admin=True) CLUSTER = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID, From 4307295de2b49e9269c2f9fed44b6707a9c98126 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sun, 28 Feb 2016 15:57:07 -0800 Subject: [PATCH 61/65] Adding upstream changes. --- gcloud_bigtable/_non_upstream_helpers.py | 5 + gcloud_bigtable/happybase/batch.py | 29 ++- gcloud_bigtable/happybase/table.py | 280 +++++++++++++---------- 3 files changed, 181 insertions(+), 133 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index 4d053e7..b828e4c 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -338,3 +338,8 @@ def _pb_timestamp_to_datetime(timestamp): microseconds=(timestamp.nanos / 1000.0), ) ) + + +def _total_seconds(timestamp): + # A fake Py2.6 backport (we don't need it since only in Py2.7.) + return timestamp.total_seconds() diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py index 78fc516..333cd06 100644 --- a/gcloud_bigtable/happybase/batch.py +++ b/gcloud_bigtable/happybase/batch.py @@ -40,6 +40,14 @@ class Batch(object): """Batch class for accumulating mutations. + .. note:: + + When using a batch with ``transaction=False`` as a context manager + (i.e. in a ``with`` statement), mutations will still be sent as + row mutations even if the context manager exits with an error. + This behavior is in place to match the behavior in the HappyBase + HBase / Thrift implementation. + :type table: :class:`Table ` :param table: The table where mutations will be applied. @@ -151,13 +159,9 @@ def put(self, row, data, wal=_WAL_SENTINEL): instance). Provided for compatibility with HappyBase, but irrelevant for Cloud Bigtable since it does not have a Write Ahead Log. - - :raises: :class:`ValueError ` if ``wal`` - is used. """ if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') + _WARN(_WAL_WARNING) row_object = self._get_row(row) # Make sure all the keys are valid before beginning @@ -224,13 +228,11 @@ def delete(self, row, columns=None, wal=_WAL_SENTINEL): irrelevant for Cloud Bigtable since it does not have a Write Ahead Log. - :raises: :class:`ValueError ` if ``wal`` - is used, or if if the delete timestamp range is set on the + :raises: If if the delete timestamp range is set on the current batch, but a full row delete is attempted. """ if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') + _WARN(_WAL_WARNING) row_object = self._get_row(row) @@ -308,9 +310,12 @@ def _get_column_pairs(columns, require_qualifier=False): """ column_pairs = [] for column in columns: + if isinstance(column, six.binary_type): + column = column.decode('utf-8') # Remove trailing colons (i.e. for standalone column family). - column = column.rstrip(':') - num_colons = column.count(':') + if column.endswith(u':'): + column = column[:-1] + num_colons = column.count(u':') if num_colons == 0: # column is a column family. if require_qualifier: @@ -319,7 +324,7 @@ def _get_column_pairs(columns, require_qualifier=False): else: column_pairs.append([column, None]) elif num_colons == 1: - column_pairs.append(column.split(':')) + column_pairs.append(column.split(u':')) else: raise ValueError('Column contains the : separator more than once') diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index 48a2cfb..a39f774 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -23,6 +23,7 @@ from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds from gcloud_bigtable._non_upstream_helpers import _to_bytes +from gcloud_bigtable._non_upstream_helpers import _total_seconds from gcloud_bigtable.column_family import GCRuleIntersection from gcloud_bigtable.column_family import MaxAgeGCRule from gcloud_bigtable.column_family import MaxVersionsGCRule @@ -99,113 +100,6 @@ def make_ordered_row(sorted_columns, include_timestamp): sorted_columns, include_timestamp) -def _filter_chain_helper(column=None, versions=None, timestamp=None, - filters=None): - """Create filter chain to limit a results set. - - :type column: str - :param column: (Optional) The column (``fam:col``) to be selected - with the filter. - - :type versions: int - :param versions: (Optional) The maximum number of cells to return. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be matched. - - :type filters: list - :param filters: (Optional) List of existing filters to be extended. - - :rtype: :class:`.RowFilter` - :returns: The chained filter created, or just a single filter if only - one was needed. - :raises: :class:`ValueError ` if there are no - filters to chain. - """ - if filters is None: - filters = [] - - if column is not None: - column_family_id, column_qualifier = column.split(':') - fam_filter = FamilyNameRegexFilter(column_family_id) - qual_filter = ColumnQualifierRegexFilter(column_qualifier) - filters.extend([fam_filter, qual_filter]) - if versions is not None: - filters.append(CellsColumnLimitFilter(versions)) - time_range = _convert_to_time_range(timestamp=timestamp) - if time_range is not None: - filters.append(TimestampRangeFilter(time_range)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterChain(filters=filters) - - -def _columns_filter_helper(columns): - """Creates a union filter for a list of columns. - - :type columns: list - :param columns: Iterable containing column names (as strings). Each column - name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :rtype: :class:`.RowFilter` - :returns: The union filter created containing all of the matched columns. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for column_family_id, column_qualifier in _get_column_pairs(columns): - fam_filter = FamilyNameRegexFilter(column_family_id) - if column_qualifier is not None: - qual_filter = ColumnQualifierRegexFilter(column_qualifier) - combined_filter = RowFilterChain( - filters=[fam_filter, qual_filter]) - filters.append(combined_filter) - else: - filters.append(fam_filter) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) - - -def _row_keys_filter_helper(row_keys): - """Creates a union filter for a list of rows. - - :type row_keys: list - :param row_keys: Iterable containing row keys (as strings). - - :rtype: :class:`.RowFilter` - :returns: The union filter created containing all of the row keys. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for row_key in row_keys: - filters.append(RowKeyRegexFilter(row_key)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) - - class Table(object): """Representation of Cloud Bigtable table. @@ -715,11 +609,25 @@ def counter_inc(self, row, column, value=1): column = column.decode('utf-8') column_family_id, column_qualifier = column.split(':') row.increment_cell_value(column_family_id, column_qualifier, value) + # See row.commit_modifications() will return a dictionary: + # { + # u'col-fam-id': { + # b'col-name1': [ + # (b'cell-val', datetime.datetime(...)), + # ... + # ], + # ... + # }, + # } modified_cells = row.commit_modifications() + # Get the cells in the modified column, column_cells = modified_cells[column_family_id][column_qualifier] + # Make sure there is exactly one cell in the column. if len(column_cells) != 1: raise ValueError('Expected server to return one modified cell.') - bytes_value = column_cells[0][0] + column_cell = column_cells[0] + # Get the bytes value from the column and convert it to an integer. + bytes_value = column_cell[0] int_value, = _UNPACK_I64(bytes_value) return int_value @@ -775,7 +683,7 @@ def _gc_rule_to_dict(gc_rule): if gc_rule is None: result = {} elif isinstance(gc_rule, MaxAgeGCRule): - result = {'time_to_live': gc_rule.max_age.total_seconds()} + result = {'time_to_live': _total_seconds(gc_rule.max_age)} elif isinstance(gc_rule, MaxVersionsGCRule): result = {'max_versions': gc_rule.max_num_versions} elif isinstance(gc_rule, GCRuleIntersection): @@ -805,10 +713,7 @@ def _next_char(str_val, index): :returns: The next character after the character at ``index`` in ``str_val``. """ - if six.PY3: # pragma: NO COVER - ord_val = str_val[index] - else: - ord_val = ord(str_val[index]) + ord_val = six.indexbytes(str_val, index) return _to_bytes(chr(ord_val + 1), encoding='latin-1') @@ -836,12 +741,8 @@ def _string_successor(str_val): index = len(str_val) - 1 while index >= 0: - if six.PY3: # pragma: NO COVER - if str_val[index] != 0xff: - break - else: - if str_val[index] != b'\xff': - break + if six.indexbytes(str_val, index) != 0xff: + break index -= 1 if index == -1: @@ -876,6 +777,17 @@ def _convert_to_time_range(timestamp=None): def _cells_to_pairs(cells, include_timestamp=False): """Converts list of cells to HappyBase format. + For example:: + + >>> import datetime + >>> from gcloud_bigtable.row_data import Cell + >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) + >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) + >>> _cells_to_pairs([cell1, cell2]) + [b'val1', b'val2'] + >>> _cells_to_pairs([cell1, cell2], include_timestamp=True) + [(b'val1', 1456361486255), (b'val2', 1456361491927)] + :type cells: list :param cells: List of :class:`.Cell` returned from a read request. @@ -902,8 +814,25 @@ def _cells_to_pairs(cells, include_timestamp=False): def _partial_row_to_dict(partial_row_data, include_timestamp=False): """Convert a low-level row data object to a dictionary. - Assumes only the latest value in each row is needed (e.g. different - behavior than in :meth:`Table.cells`). + Assumes only the latest value in each row is needed. This assumption + is due to the fact that this method is used by callers which use + a ``CellsColumnLimitFilter(1)`` filter. + + For example:: + + >>> import datetime + >>> from gcloud_bigtable.row_data import Cell, PartialRowData + >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) + >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) + >>> row_data = PartialRowData(b'row-key') + >>> _partial_row_to_dict(row_data) + {} + >>> row_data._cells[u'fam1'] = {b'col1': [cell1], b'col2': [cell2]} + >>> _partial_row_to_dict(row_data) + {b'fam1:col2': b'val2', b'fam1:col1': b'val1'} + >>> _partial_row_to_dict(row_data, include_timestamp=True) + {b'fam1:col2': (b'val2', 1456361724480), + b'fam1:col1': (b'val1', 1456361721135)} :type partial_row_data: :class:`.row_data.PartialRowData` :param partial_row_data: Row data consumed from a stream. @@ -923,3 +852,112 @@ def _partial_row_to_dict(partial_row_data, include_timestamp=False): # our filter, but we don't check this. result[column] = cell_vals[0] return result + + +def _filter_chain_helper(column=None, versions=None, timestamp=None, + filters=None): + """Create filter chain to limit a results set. + + :type column: str + :param column: (Optional) The column (``fam:col``) to be selected + with the filter. + + :type versions: int + :param versions: (Optional) The maximum number of cells to return. + + :type timestamp: int + :param timestamp: (Optional) Timestamp (in milliseconds since the + epoch). If specified, only cells returned before (or + at) the timestamp will be matched. + + :type filters: list + :param filters: (Optional) List of existing filters to be extended. + + :rtype: :class:`.RowFilter` + :returns: The chained filter created, or just a single filter if only + one was needed. + :raises: :class:`ValueError ` if there are no + filters to chain. + """ + if filters is None: + filters = [] + + if column is not None: + if isinstance(column, six.binary_type): + column = column.decode('utf-8') + column_family_id, column_qualifier = column.split(':') + fam_filter = FamilyNameRegexFilter(column_family_id) + qual_filter = ColumnQualifierRegexFilter(column_qualifier) + filters.extend([fam_filter, qual_filter]) + if versions is not None: + filters.append(CellsColumnLimitFilter(versions)) + time_range = _convert_to_time_range(timestamp=timestamp) + if time_range is not None: + filters.append(TimestampRangeFilter(time_range)) + + num_filters = len(filters) + if num_filters == 0: + raise ValueError('Must have at least one filter.') + elif num_filters == 1: + return filters[0] + else: + return RowFilterChain(filters=filters) + + +def _columns_filter_helper(columns): + """Creates a union filter for a list of columns. + + :type columns: list + :param columns: Iterable containing column names (as strings). Each column + name can be either + + * an entire column family: ``fam`` or ``fam:`` + * an single column: ``fam:col`` + + :rtype: :class:`.RowFilter` + :returns: The union filter created containing all of the matched columns. + :raises: :class:`ValueError ` if there are no + filters to union. + """ + filters = [] + for column_family_id, column_qualifier in _get_column_pairs(columns): + fam_filter = FamilyNameRegexFilter(column_family_id) + if column_qualifier is not None: + qual_filter = ColumnQualifierRegexFilter(column_qualifier) + combined_filter = RowFilterChain( + filters=[fam_filter, qual_filter]) + filters.append(combined_filter) + else: + filters.append(fam_filter) + + num_filters = len(filters) + if num_filters == 0: + raise ValueError('Must have at least one filter.') + elif num_filters == 1: + return filters[0] + else: + return RowFilterUnion(filters=filters) + + +def _row_keys_filter_helper(row_keys): + """Creates a union filter for a list of rows. + + :type row_keys: list + :param row_keys: Iterable containing row keys (as strings). + + :rtype: :class:`.RowFilter` + :returns: The union filter created containing all of the row keys. + :raises: :class:`ValueError ` if there are no + filters to union. + """ + filters = [] + for row_key in row_keys: + filters.append(RowKeyRegexFilter(row_key)) + + num_filters = len(filters) + if num_filters == 0: + raise ValueError('Must have at least one filter.') + elif num_filters == 1: + return filters[0] + else: + return RowFilterUnion(filters=filters) From dc05d25293039ded7e56b625836974e2ca708e89 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Sun, 28 Feb 2016 16:06:33 -0800 Subject: [PATCH 62/65] Fixing some broken tests. --- gcloud_bigtable/_non_upstream_helpers.py | 5 ---- gcloud_bigtable/happybase/table.py | 2 +- gcloud_bigtable/happybase/test_batch.py | 36 ++++++++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py index b828e4c..4d053e7 100644 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ b/gcloud_bigtable/_non_upstream_helpers.py @@ -338,8 +338,3 @@ def _pb_timestamp_to_datetime(timestamp): microseconds=(timestamp.nanos / 1000.0), ) ) - - -def _total_seconds(timestamp): - # A fake Py2.6 backport (we don't need it since only in Py2.7.) - return timestamp.total_seconds() diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py index a39f774..86046b0 100644 --- a/gcloud_bigtable/happybase/table.py +++ b/gcloud_bigtable/happybase/table.py @@ -883,7 +883,7 @@ def _filter_chain_helper(column=None, versions=None, timestamp=None, filters = [] if column is not None: - if isinstance(column, six.binary_type): + if isinstance(column, six.binary_type): # pragma: NO COVER column = column.decode('utf-8') column_family_id, column_qualifier = column.split(':') fam_filter = FamilyNameRegexFilter(column_family_id) diff --git a/gcloud_bigtable/happybase/test_batch.py b/gcloud_bigtable/happybase/test_batch.py index 88e2ebc..69f81a4 100644 --- a/gcloud_bigtable/happybase/test_batch.py +++ b/gcloud_bigtable/happybase/test_batch.py @@ -36,7 +36,7 @@ def test_it(self): expected_result = [ ['cf1', None], ['cf2', None], - ['cf3', None], + ['cf3', ''], ['cf3', 'name1'], ['cf3', 'name2'], ] @@ -241,7 +241,14 @@ def test__get_row_create_new(self): self.assertEqual(batch._row_map, {row_key: mock_row}) def test_put_bad_wal(self): - from gcloud_bigtable.happybase.batch import _WAL_SENTINEL + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import batch as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + raise RuntimeError('No need to continue') table = object() batch = self._makeOne(table) @@ -250,9 +257,11 @@ def test_put_bad_wal(self): data = {} wal = None - self.assertNotEqual(wal, _WAL_SENTINEL) - with self.assertRaises(ValueError): - batch.put(row, data, wal=wal) + self.assertNotEqual(wal, MUT._WAL_SENTINEL) + with _Monkey(MUT, _WARN=mock_warn): + with self.assertRaises(RuntimeError): + batch.put(row, data, wal=wal) + self.assertEqual(warned, [MUT._WAL_WARNING]) def test_put(self): import operator @@ -345,7 +354,14 @@ def test__delete_columns_w_time_and_col_fam(self): self._delete_columns_test_helper(time_range=time_range) def test_delete_bad_wal(self): - from gcloud_bigtable.happybase.batch import _WAL_SENTINEL + from gcloud_bigtable._testing import _Monkey + from gcloud_bigtable.happybase import batch as MUT + + warned = [] + + def mock_warn(msg): + warned.append(msg) + raise RuntimeError('No need to continue') table = object() batch = self._makeOne(table) @@ -354,9 +370,11 @@ def test_delete_bad_wal(self): columns = [] wal = None - self.assertNotEqual(wal, _WAL_SENTINEL) - with self.assertRaises(ValueError): - batch.delete(row, columns=columns, wal=wal) + self.assertNotEqual(wal, MUT._WAL_SENTINEL) + with _Monkey(MUT, _WARN=mock_warn): + with self.assertRaises(RuntimeError): + batch.delete(row, columns=columns, wal=wal) + self.assertEqual(warned, [MUT._WAL_WARNING]) def test_delete_entire_row(self): table = object() From d3789abbb95a97f3debc09955aa53c06d1871fcf Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 29 Feb 2016 11:57:30 -0800 Subject: [PATCH 63/65] Adding old notes. --- gcloud_bigtable/happybase/notes.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gcloud_bigtable/happybase/notes.txt b/gcloud_bigtable/happybase/notes.txt index 183e144..bdc5af1 100644 --- a/gcloud_bigtable/happybase/notes.txt +++ b/gcloud_bigtable/happybase/notes.txt @@ -80,3 +80,19 @@ print(list(table.scan())) print(list(table.scan(sorted_columns=True))) + + + + +TESTING IF DELETE TIMESTAMPS MATTER +table.cells(row_key, 'family:qual1', include_timestamp=True) +table.put(row_key, {'family:qual1': 'new-data'}) +table.cells(row_key, 'family:qual1', include_timestamp=True) +table.delete(row_key, columns=['family:qual1']) +table.cells(row_key, 'family:qual1', include_timestamp=True) +table.put(row_key, {'family:qual1': 'value1'}) +table.put(row_key, {'family:qual1': 'new-data'}) +c1, c2 = table.cells(row_key, 'family:qual1', include_timestamp=True) +midpt = (c1[1] + c2[1]) // 2 +table.delete(row_key, columns=['family:qual1'], timestamp=midpt) +table.cells(row_key, 'family:qual1', include_timestamp=True) From 7355f985ded6e5406731d3e74845fda8b8daf083 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 17 May 2016 09:03:42 -0700 Subject: [PATCH 64/65] Retiring this project. --- .coveragerc | 6 - .gitignore | 46 - CONTRIBUTING.md | 78 - MANIFEST.in | 3 - Makefile | 67 - README.md | 148 +- _fake_grpc/grpc/__init__.py | 13 - _fake_grpc/grpc/_adapter/__init__.py | 13 - _fake_grpc/grpc/_adapter/_c.py | 13 - _fake_grpc/grpc/beta/__init__.py | 13 - _fake_grpc/grpc/beta/implementations.py | 13 - _fake_grpc/grpc/beta/interfaces.py | 13 - _fake_grpc/grpc/early_adopter/__init__.py | 13 - .../grpc/early_adopter/implementations.py | 13 - _fake_grpc/grpc/framework/__init__.py | 13 - _fake_grpc/grpc/framework/alpha/__init__.py | 13 - _fake_grpc/grpc/framework/alpha/utilities.py | 13 - _fake_grpc/grpc/framework/common/__init__.py | 13 - .../grpc/framework/common/cardinality.py | 13 - .../grpc/framework/interfaces/__init__.py | 13 - .../framework/interfaces/face/__init__.py | 13 - .../grpc/framework/interfaces/face/face.py | 13 - .../framework/interfaces/face/utilities.py | 13 - docs/client-intro.rst | 102 - docs/client.rst | 7 - docs/cluster-api.rst | 179 -- docs/cluster.rst | 7 - docs/column-family.rst | 48 - docs/data-api.rst | 327 ---- docs/happybase-batch.rst | 7 - docs/happybase-connection.rst | 7 - docs/happybase-package.rst | 7 - docs/happybase-pool.rst | 7 - docs/happybase-table.rst | 7 - docs/index.rst | 80 +- docs/row-data.rst | 7 - docs/row.rst | 7 - docs/table-api.rst | 172 -- docs/table.rst | 7 - gcloud_bigtable/__init__.py | 18 - gcloud_bigtable/_generated/__init__.py | 15 - .../_generated/_bigtable_cluster_data.proto | 93 - .../_bigtable_cluster_service.proto | 129 -- .../_bigtable_cluster_service_messages.proto | 134 -- .../_generated/_bigtable_data.proto | 515 ----- .../_generated/_bigtable_service.proto | 73 - .../_bigtable_service_messages.proto | 214 -- .../_generated/_bigtable_table_data.proto | 125 -- .../_generated/_bigtable_table_service.proto | 74 - .../_bigtable_table_service_messages.proto | 101 - .../_generated/bigtable_cluster_data_pb2.py | 221 --- .../bigtable_cluster_service_messages_pb2.py | 538 ----- .../bigtable_cluster_service_pb2.py | 187 -- .../_generated/bigtable_data_pb2.py | 1226 ------------ .../bigtable_service_messages_pb2.py | 678 ------- .../_generated/bigtable_service_pb2.py | 167 -- .../_generated/bigtable_table_data_pb2.py | 377 ---- .../bigtable_table_service_messages_pb2.py | 389 ---- .../_generated/bigtable_table_service_pb2.py | 203 -- .../_generated/operations_grpc_pb2.py | 100 - gcloud_bigtable/_grpc_mocks.py | 47 - gcloud_bigtable/_non_upstream_helpers.py | 340 ---- gcloud_bigtable/_testing.py | 78 - gcloud_bigtable/client.py | 475 ----- gcloud_bigtable/cluster.py | 482 ----- gcloud_bigtable/column_family.py | 326 ---- gcloud_bigtable/happybase/__init__.py | 160 -- gcloud_bigtable/happybase/batch.py | 331 ---- gcloud_bigtable/happybase/connection.py | 459 ----- gcloud_bigtable/happybase/notes.txt | 98 - gcloud_bigtable/happybase/pool.py | 151 -- gcloud_bigtable/happybase/table.py | 963 --------- gcloud_bigtable/happybase/test_batch.py | 559 ------ gcloud_bigtable/happybase/test_connection.py | 608 ------ gcloud_bigtable/happybase/test_pool.py | 257 --- gcloud_bigtable/happybase/test_table.py | 1363 ------------- gcloud_bigtable/row.py | 1339 ------------- gcloud_bigtable/row_data.py | 328 ---- gcloud_bigtable/table.py | 423 ---- gcloud_bigtable/test__non_upstream_helpers.py | 416 ---- gcloud_bigtable/test_client.py | 764 -------- gcloud_bigtable/test_cluster.py | 761 -------- gcloud_bigtable/test_column_family.py | 627 ------ gcloud_bigtable/test_row.py | 1735 ----------------- gcloud_bigtable/test_row_data.py | 527 ----- gcloud_bigtable/test_table.py | 574 ------ pylintrc_default | 416 ---- run_pylint.py | 238 --- scripts/check_generate.py | 37 - scripts/make_operations_grpc.py | 116 -- scripts/pep8_on_repo.sh | 19 - scripts/rewrite_imports.py | 143 -- setup.py | 48 - system_tests/local_setup.sample | 3 - system_tests/run.py | 457 ----- system_tests/run_happybase.py | 997 ---------- tox.ini | 61 - 97 files changed, 7 insertions(+), 23843 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .gitignore delete mode 100644 CONTRIBUTING.md delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 _fake_grpc/grpc/__init__.py delete mode 100644 _fake_grpc/grpc/_adapter/__init__.py delete mode 100644 _fake_grpc/grpc/_adapter/_c.py delete mode 100644 _fake_grpc/grpc/beta/__init__.py delete mode 100644 _fake_grpc/grpc/beta/implementations.py delete mode 100644 _fake_grpc/grpc/beta/interfaces.py delete mode 100644 _fake_grpc/grpc/early_adopter/__init__.py delete mode 100644 _fake_grpc/grpc/early_adopter/implementations.py delete mode 100644 _fake_grpc/grpc/framework/__init__.py delete mode 100644 _fake_grpc/grpc/framework/alpha/__init__.py delete mode 100644 _fake_grpc/grpc/framework/alpha/utilities.py delete mode 100644 _fake_grpc/grpc/framework/common/__init__.py delete mode 100644 _fake_grpc/grpc/framework/common/cardinality.py delete mode 100644 _fake_grpc/grpc/framework/interfaces/__init__.py delete mode 100644 _fake_grpc/grpc/framework/interfaces/face/__init__.py delete mode 100644 _fake_grpc/grpc/framework/interfaces/face/face.py delete mode 100644 _fake_grpc/grpc/framework/interfaces/face/utilities.py delete mode 100644 docs/client-intro.rst delete mode 100644 docs/client.rst delete mode 100644 docs/cluster-api.rst delete mode 100644 docs/cluster.rst delete mode 100644 docs/column-family.rst delete mode 100644 docs/data-api.rst delete mode 100644 docs/happybase-batch.rst delete mode 100644 docs/happybase-connection.rst delete mode 100644 docs/happybase-package.rst delete mode 100644 docs/happybase-pool.rst delete mode 100644 docs/happybase-table.rst delete mode 100644 docs/row-data.rst delete mode 100644 docs/row.rst delete mode 100644 docs/table-api.rst delete mode 100644 docs/table.rst delete mode 100644 gcloud_bigtable/__init__.py delete mode 100644 gcloud_bigtable/_generated/__init__.py delete mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_data.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_service.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_data.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_service.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_service_messages.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_table_data.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_table_service.proto delete mode 100644 gcloud_bigtable/_generated/_bigtable_table_service_messages.proto delete mode 100644 gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_cluster_service_messages_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_cluster_service_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_data_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_service_messages_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_service_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_table_data_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py delete mode 100644 gcloud_bigtable/_generated/bigtable_table_service_pb2.py delete mode 100644 gcloud_bigtable/_generated/operations_grpc_pb2.py delete mode 100644 gcloud_bigtable/_grpc_mocks.py delete mode 100644 gcloud_bigtable/_non_upstream_helpers.py delete mode 100644 gcloud_bigtable/_testing.py delete mode 100644 gcloud_bigtable/client.py delete mode 100644 gcloud_bigtable/cluster.py delete mode 100644 gcloud_bigtable/column_family.py delete mode 100644 gcloud_bigtable/happybase/__init__.py delete mode 100644 gcloud_bigtable/happybase/batch.py delete mode 100644 gcloud_bigtable/happybase/connection.py delete mode 100644 gcloud_bigtable/happybase/notes.txt delete mode 100644 gcloud_bigtable/happybase/pool.py delete mode 100644 gcloud_bigtable/happybase/table.py delete mode 100644 gcloud_bigtable/happybase/test_batch.py delete mode 100644 gcloud_bigtable/happybase/test_connection.py delete mode 100644 gcloud_bigtable/happybase/test_pool.py delete mode 100644 gcloud_bigtable/happybase/test_table.py delete mode 100644 gcloud_bigtable/row.py delete mode 100644 gcloud_bigtable/row_data.py delete mode 100644 gcloud_bigtable/table.py delete mode 100644 gcloud_bigtable/test__non_upstream_helpers.py delete mode 100644 gcloud_bigtable/test_client.py delete mode 100644 gcloud_bigtable/test_cluster.py delete mode 100644 gcloud_bigtable/test_column_family.py delete mode 100644 gcloud_bigtable/test_row.py delete mode 100644 gcloud_bigtable/test_row_data.py delete mode 100644 gcloud_bigtable/test_table.py delete mode 100644 pylintrc_default delete mode 100644 run_pylint.py delete mode 100644 scripts/check_generate.py delete mode 100644 scripts/make_operations_grpc.py delete mode 100755 scripts/pep8_on_repo.sh delete mode 100644 scripts/rewrite_imports.py delete mode 100644 setup.py delete mode 100644 system_tests/local_setup.sample delete mode 100644 system_tests/run.py delete mode 100644 system_tests/run_happybase.py delete mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 86f47e7..0000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[report] -omit = - */_generated/*.py -exclude_lines = - # Re-enable the standard pragma - pragma: NO COVER diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 925072d..0000000 --- a/.gitignore +++ /dev/null @@ -1,46 +0,0 @@ -*.py[cod] -*.sw[op] - -# C extensions -*.so - -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 -__pycache__ - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.tox -nosetests.xml - -# Virtual environment -env/ -coverage.xml - -# Directories used for creating generated PB2 files -generated_python/ -cloud-bigtable-client/ - -# Make sure a generated file isn't accidentally committed. -pylintrc_reduced - -# Local config not intended to be uploaded -system_tests/local_setup - -# Sphinx built docs -docs/_build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 1396f1d..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,78 +0,0 @@ -## Building `_pb2.py` files - -The `gcloud_bigtable/_generated` directory depends on files -generated from [`bigtable-protos`][1]. These generated -files must be re-generated from time to time. - -To generate them, execute - -```bash -make generate -``` - -Compiling them to Python files requires -- the latest version of the `protoc` compiler to support the `proto3` syntax -- the `grpc_python_plugin` (which requires installing `grpc`) -- the latest version of the Python [`protobuf`][3] library: - - ```bash - $ [sudo] pip install "protobuf>=3.0.0a3" - ``` - -## Installing `protoc>=3.0.0` - -In order to install `protoc` version `>= 3.0.0`, follow the -[instructions][2] on the project. For Debian-based Linux, they are roughly - -```bash -$ sudo apt-get install autoconf libtool -$ git clone https://github.com/google/protobuf -$ cd protobuf -$ # Optionally checkout a tagged commit: git checkout v3.0.0-alpha-3.1 -$ ./autogen.sh # Generate the configure script -$ # Build and install C++ Protocol Buffer runtime and -$ # the Protocol Buffer compiler (protoc) -$ ./configure [--prefix=/usr/local] # or [--prefix=/usr] -$ make -$ make check -$ # Make sure that `make check` passes -$ [sudo] make install -$ # May need to update dynamic linker config via: -$ # [sudo] ldconfig -``` - -## Testing - -`tox` should work without modification, assuming you've installed -on Linux via - -```bash -$ apt-get install libgrpc-dev -``` - -or on Mac OS X via - -```bash -$ brew tap grpc/grpc -$ brew install grpc -``` - -## False Starts - -This library originally attempted to support HTTP-RPC requests -over HTTP/1.1. This is not allowed by the backend, and results in -a 400 Bad Request: - -```json -{ - "error": { - "code": 400, - "message": "Proto over HTTP is not allowed for service 'bigtableclusteradmin.googleapis.com'.", - "status": "FAILED_PRECONDITION" - } -} -``` - -[1]: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/ -[2]: https://github.com/google/protobuf -[3]: https://pypi.python.org/pypi/protobuf diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 36ebf02..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.md -graft gcloud_bigtable -global-exclude *.pyc diff --git a/Makefile b/Makefile deleted file mode 100644 index c25b6fc..0000000 --- a/Makefile +++ /dev/null @@ -1,67 +0,0 @@ -GENERATED_DIR=$(shell pwd)/generated_python -FINAL_DIR=$(shell pwd)/gcloud_bigtable/_generated -GRPC_PLUGIN=grpc_python_plugin -PROTOC_CMD=protoc -PROTOS_DIR=$(shell pwd)/cloud-bigtable-client/bigtable-protos/src/main/proto - -help: - @echo 'Makefile for a gcloud-python-bigtable ' - @echo ' ' - @echo ' make generate Generates the protobuf modules ' - @echo ' make check_generate Checks that generate succeeded ' - @echo ' make clean Clean generated files ' - -generate: - [ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client - cd cloud-bigtable-client && git pull origin master - mkdir -p $(GENERATED_DIR) - # Generate all *_pb2.py files that require gRPC. - $(PROTOC_CMD) \ - --proto_path=$(PROTOS_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - --python_out=$(GENERATED_DIR) \ - --grpc_out=$(GENERATED_DIR) \ - $(PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \ - $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto - # Generate all *_pb2.py files that do not require gRPC. - $(PROTOC_CMD) \ - --proto_path=$(PROTOS_DIR) \ - --python_out=$(GENERATED_DIR) \ - $(PROTOS_DIR)/google/bigtable/v1/bigtable_data.proto \ - $(PROTOS_DIR)/google/bigtable/v1/bigtable_service_messages.proto \ - $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto \ - $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \ - $(PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto - # Move the newly generated *_pb2.py files into our library. - mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) - mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(FINAL_DIR) - mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(FINAL_DIR) - # Remove all existing *.proto files before we replace - rm -f $(FINAL_DIR)/*.proto - # Copy over the *.proto files into our library. - cp $(PROTOS_DIR)/google/bigtable/v1/*.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(FINAL_DIR) - cp $(PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(FINAL_DIR) - # Rename all *.proto files in our library with an - # underscore and remove executable bit. - cd $(FINAL_DIR) && \ - for filename in *.proto; do \ - chmod -x $$filename ; \ - mv $$filename _$$filename ; \ - done - # Separate the gRPC parts of the operations service from the - # non-gRPC parts so that the protos from `googleapis-common-protos` - # can be used without gRPC. - python scripts/make_operations_grpc.py - # Rewrite the imports in the generated *_pb2.py files. - python scripts/rewrite_imports.py - -check_generate: - python scripts/check_generate.py - -clean: - rm -fr cloud-bigtable-client $(GENERATED_DIR) - -.PHONY: generate check_generate clean diff --git a/README.md b/README.md index f8e70a6..4190617 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,7 @@ # Google Cloud Bigtable Python Library (Alpha) -[![Documentation Status](https://readthedocs.org/projects/gcloud-python-bigtable/badge/?version=latest)](https://readthedocs.org/projects/gcloud-python-bigtable/?badge=latest) +This library was an alpha implementation of [Google Cloud Bigtable][1]. +The project has been moved into [`gcloud-python`][2]. -### An extension to [`gcloud-python`][1] - -This library supports RPC requests to the Google Cloud Bigtable API over -HTTP/2. In order to support this, we'll rely on [`grpc`][2]. Unfortunately, -the install story of gRPC is still developing. We are working with the -gRPC team to rapidly make the install story more user-friendly. - -The Cloud Bigtable API is supported via JSON over HTTP/1.1 (see the -[`gcloud` CLI][4] use of the API). However, features of HTTP/2 such -as streaming are [used][5] by the Bigtable API and are not possible -to support via HTTP/1.1. - -## Installing gRPC - -Before you can install the Python library (`grpcio`) you'll need the -gRPC core library on your system. To install this on Debian Linux: - -```bash -$ apt-get install libgrpc-dev -``` - -or on Mac OS X: - -```bash -$ brew tap grpc/grpc -$ brew install grpc -``` - -Once you've installed the system library, install the Python -library via: - -```bash -$ pip install --upgrade grpcio -``` - -Finally, you can install this library via - -```bash -pip install -e git+https://github.com/dhermes/gcloud-python-bigtable#egg=gcloud-bigtable -``` - -## Authorization - -You can make requests with your own Google account by -using the [`gcloud` CLI tool][8]. You can create an access token via - -```bash -gcloud login -``` - -and then from there, the token created will be picked up automatically -when you create an object which requires authentication: - -```python -from gcloud_bigtable.client import Client -client = Client() -``` - -If instead you'd like to use a service account, you can set an -environment variable to the path containing the service account -credentials JSON file: - -```bash -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json" -``` - -If you are **familiar** with the [`oauth2client`][9] library, -you can create a `credentials` object and pass it directly: - -```python -from gcloud_bigtable.client import Client -client = Client(credentials=credentials) -``` - -## Enabling the Bigtable API - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"APIs & auth > APIs"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/apiui/apis/library - ``` - -1. On this page, search for **bigtable**, and click both `Cloud Bigtable API` - and `Cloud Bigtable Table Admin API`. -1. For each API, click "Enable API" (if not already enabled) - -## Getting a Service Account Keyfile - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"APIs & auth > Credentials"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/apiui/credential - ``` - -1. On this page, click "Create new Client ID", select "Service account" as - your "Application type" and then download the JSON key provided. - -After downloading, you can use this file as your -`GOOGLE_APPLICATION_CREDENTIALS`. - -## Creating a Cluster in the UI - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"Storage > Cloud Bigtable"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/bigtable/clusters - ``` - -1. On this page, click **Create a cluster** and take note of the "Cluster ID" - and "Zone" you use when creating it. - -## Error Messages - -Unfortunately, the gRPC Python library does not surface -exceptions returned from the API. An [issue][10] has been -filed with the gRPC team about this problem. - -## Development - -See [`CONTRIBUTING.md`][3] for instructions on development. - -[1]: https://github.com/GoogleCloudPlatform/gcloud-python -[2]: https://www.grpc.io/ -[3]: https://github.com/dhermes/gcloud-python-bigtable/blob/master/CONTRIBUTING.md -[4]: https://cloud.google.com/sdk/gcloud/reference/alpha/bigtable/clusters/list -[5]: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L46 -[6]: http://brew.sh/ -[7]: https://github.com/Homebrew/linuxbrew#install-linuxbrew-tldr -[8]: https://cloud.google.com/sdk/gcloud/ -[9]: https://pypi.python.org/pypi/oauth2client -[10]: https://github.com/grpc/grpc/issues/2611 -[11]: https://github.com/grpc/grpc/tree/master/src/python -[12]: https://pypi.python.org/pypi/grpcio -[14]: https://console.developers.google.com/ +[1]: https://cloud.google.com/bigtable/docs/ +[2]: http://gcloud-python.readthedocs.io/en/stable/bigtable-usage.html diff --git a/_fake_grpc/grpc/__init__.py b/_fake_grpc/grpc/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/_adapter/__init__.py b/_fake_grpc/grpc/_adapter/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/_adapter/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/_adapter/_c.py b/_fake_grpc/grpc/_adapter/_c.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/_adapter/_c.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/beta/__init__.py b/_fake_grpc/grpc/beta/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/beta/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/beta/implementations.py b/_fake_grpc/grpc/beta/implementations.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/beta/implementations.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/beta/interfaces.py b/_fake_grpc/grpc/beta/interfaces.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/beta/interfaces.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/early_adopter/__init__.py b/_fake_grpc/grpc/early_adopter/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/early_adopter/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/early_adopter/implementations.py b/_fake_grpc/grpc/early_adopter/implementations.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/early_adopter/implementations.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/__init__.py b/_fake_grpc/grpc/framework/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/alpha/__init__.py b/_fake_grpc/grpc/framework/alpha/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/alpha/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/alpha/utilities.py b/_fake_grpc/grpc/framework/alpha/utilities.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/alpha/utilities.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/common/__init__.py b/_fake_grpc/grpc/framework/common/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/common/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/common/cardinality.py b/_fake_grpc/grpc/framework/common/cardinality.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/common/cardinality.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/__init__.py b/_fake_grpc/grpc/framework/interfaces/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/interfaces/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/__init__.py b/_fake_grpc/grpc/framework/interfaces/face/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/interfaces/face/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/face.py b/_fake_grpc/grpc/framework/interfaces/face/face.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/interfaces/face/face.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/interfaces/face/utilities.py b/_fake_grpc/grpc/framework/interfaces/face/utilities.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/interfaces/face/utilities.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/docs/client-intro.rst b/docs/client-intro.rst deleted file mode 100644 index 495bd01..0000000 --- a/docs/client-intro.rst +++ /dev/null @@ -1,102 +0,0 @@ -Base for Everything -=================== - -To use the API, the :class:`Client ` -class defines a high-level interface which handles authorization -and creating other objects: - -.. code:: python - - from gcloud_bigtable.client import Client - client = Client() - -Long-lived Defaults -------------------- - -When creating a :class:`Client `, the -``user_agent`` and ``timeout_seconds`` arguments have sensible -defaults -(:data:`DEFAULT_USER_AGENT ` and -:data:`DEFAULT_TIMEOUT_SECONDS `). -However, you may over-ride them and these will be used throughout all API -requests made with the ``client`` you create. - -Authorization -------------- - -This will use the Google `Application Default Credentials`_ if -you don't pass any credentials of your own. If you are **familiar** with the -`oauth2client`_ library, you can create a ``credentials`` object and -pass it directly: - -.. code:: python - - client = Client(credentials=credentials) - -In addition, the -:meth:`from_service_account_json() ` -and -:meth:`from_service_account_p12() ` -factories can be used if you know the specific type of credentials you'd -like to use. - -Project ID ----------- - -.. tip:: - - Be sure to use the **Project ID**, not the **Project Number**. - -You can also explicitly provide the ``project`` rather than relying -on the inferred value: - -.. code:: python - - client = Client(project='my-cloud-console-project') - -When implicit, the value is inferred from the environment in the following -order: - -* The ``GCLOUD_PROJECT`` environment variable -* The Google App Engine application ID -* The Google Compute Engine project ID (from the metadata server) - -Admin API Access ----------------- - -If you'll be using your client to make `Cluster Admin`_ and `Table Admin`_ -API requests, you'll need to pass the ``admin`` argument: - -.. code:: python - - client = Client(admin=True) - -Read-Only Mode --------------- - -If on the other hand, you only have (or want) read access to the data, -you can pass the ``read_only`` argument: - -.. code:: python - - client = Client(read_only=True) - -This will ensure that the -:data:`READ_ONLY_SCOPE ` is used -for API requests (so any accidental requests that would modify data will -fail). - -Next Step ---------- - -After a :class:`Client `, the next highest-level -object is a :class:`Cluster `. You'll need -one before you can interact with tables or data. - -Head next to learn about the `Cluster Admin API`_. - -.. _Application Default Credentials: https://developers.google.com/identity/protocols/application-default-credentials -.. _oauth2client: http://oauth2client.readthedocs.org/en/latest/ -.. _Cluster Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1 -.. _Table Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1 -.. _Cluster Admin API: cluster-api.html diff --git a/docs/client.rst b/docs/client.rst deleted file mode 100644 index feb8aed..0000000 --- a/docs/client.rst +++ /dev/null @@ -1,7 +0,0 @@ -Client -~~~~~~ - -.. automodule:: gcloud_bigtable.client - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/cluster-api.rst b/docs/cluster-api.rst deleted file mode 100644 index b06aca1..0000000 --- a/docs/cluster-api.rst +++ /dev/null @@ -1,179 +0,0 @@ -Cluster Admin API -================= - -After creating a :class:`Client `, you can -interact with individual clusters, groups of clusters or available -zones for a project. - -List Clusters -------------- - -If you want a comprehensive list of all existing clusters, make a -`ListClusters`_ API request with -:meth:`Client.list_clusters() `: - -.. code:: python - - clusters = client.list_clusters() - -List Zones ----------- - -If you aren't sure which ``zone`` to create a cluster in, find out -which zones your project has access to with a `ListZones`_ API request -with :meth:`Client.list_zones() `: - -.. code:: python - - zones = client.list_zones() - -You can choose a :class:`string ` from among the result to pass to -the :class:`Cluster ` constructor. - -As of right now, the available zones are - -.. code:: python - - >>> zones - [u'asia-east1-b', u'europe-west1-c', u'us-central1-c', u'us-central1-b'] - -Cluster Factory ---------------- - -To create a :class:`Cluster ` object: - -.. code:: python - - cluster = client.cluster(zone, cluster_id, - display_name=display_name, - serve_nodes=3) - -Both ``display_name`` and ``serve_nodes`` are optional. When not provided, -``display_name`` defaults to the ``cluster_id`` value and ``serve_nodes`` -defaults to the minimum allowed: ``3``. - -Even if this :class:`Cluster ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`Table ` just as the -:class:`Client ` is used as the parent of -a :class:`Cluster `. - -Create a new Cluster --------------------- - -After creating the cluster object, make a `CreateCluster`_ API request -with :meth:`create() `: - -.. code:: python - - cluster.display_name = 'My very own cluster' - cluster.create() - -If you would like more than the minimum number of nodes (``3``) in your cluster: - -.. code:: python - - cluster.serve_nodes = 10 - cluster.create() - -Check on Current Operation --------------------------- - -.. note:: - - When modifying a cluster (via a `CreateCluster`_, `UpdateCluster`_ or - `UndeleteCluster`_ request), the Bigtable API will return a long-running - `Operation`_. This will be stored on the object after each of - :meth:`create() `, - :meth:`update() ` and - :meth:`undelete() ` are called. - -.. _Operation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L73-L102 - -You can check if a long-running operation (for a -:meth:`create() `, -:meth:`update() ` or -:meth:`undelete() `) has finished -by making a `GetOperation`_ request with -:meth:`operation_finished() `: - -.. code:: python - - >>> cluster.operation_finished() - True - -.. note:: - - The operation data is stored in protected fields on the - :class:`Cluster `: - ``_operation_type``, ``_operation_id`` and ``_operation_begin``. - If these are unset, then - :meth:`operation_finished() ` - will fail. Also, these will be removed after a long-running operation - has completed (checked via this method). We could easily surface these - properties publicly, but it's unclear if end-users would need them. - -Get metadata for an existing Cluster ------------------------------------- - -After creating the cluster object, make a `GetCluster`_ API request -with :meth:`reload() `: - -.. code:: python - - cluster.reload() - -This will load ``serve_nodes`` and ``display_name`` for the existing -``cluster`` in addition to the ``cluster_id``, ``zone`` and ``project`` -already set on the :class:`Cluster ` object. - -Update an existing Cluster --------------------------- - -After creating the cluster object, make an `UpdateCluster`_ API request -with :meth:`update() `: - -.. code:: python - - client.display_name = 'New display_name' - cluster.update() - -Delete an existing Cluster --------------------------- - -Make a `DeleteCluster`_ API request with -:meth:`delete() `: - -.. code:: python - - cluster.delete() - -Undelete a deleted Cluster --------------------------- - -Make an `UndeleteCluster`_ API request with -:meth:`undelete() `: - -.. code:: python - - cluster.undelete() - -Next Step ---------- - -Now we go down the hierarchy from -:class:`Cluster ` to a -:class:`Table `. - -Head next to learn about the `Table Admin API`_. - -.. _Cluster Admin API: https://cloud.google.com/bigtable/docs/creating-cluster -.. _CreateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L66-L68 -.. _GetCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L38-L40 -.. _UpdateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L93-L95 -.. _DeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L109-L111 -.. _ListZones: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L33-L35 -.. _ListClusters: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L44-L46 -.. _GetOperation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/bfe4138f04bf3383a558152e4333112cdd13d5b0/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L43-L45 -.. _UndeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L126-L128 -.. _Table Admin API: table-api.html diff --git a/docs/cluster.rst b/docs/cluster.rst deleted file mode 100644 index d22d213..0000000 --- a/docs/cluster.rst +++ /dev/null @@ -1,7 +0,0 @@ -Cluster -~~~~~~~ - -.. automodule:: gcloud_bigtable.cluster - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/column-family.rst b/docs/column-family.rst deleted file mode 100644 index 1210803..0000000 --- a/docs/column-family.rst +++ /dev/null @@ -1,48 +0,0 @@ -Column Families -=============== - -When creating a -:class:`ColumnFamily `, it is -possible to set garbage collection rules for expired data. - -By setting a rule, cells in the table matching the rule will be deleted -during periodic garbage collection (which executes opportunistically in the -background). - -The types -:class:`GarbageCollectionRule `, -:class:`GarbageCollectionRuleUnion ` and -:class:`GarbageCollectionRuleIntersection ` -can all be used as the optional ``gc_rule`` argument in the -:class:`ColumnFamily ` -constructor. This value is then used in the -:meth:`create() ` and -:meth:`update() ` methods. - -These rules can be nested arbitrarily, with -:class:`GarbageCollectionRule ` -at the lowest level of the nesting: - -.. code:: python - - import datetime - - max_age = datetime.timedelta(days=3) - rule1 = GarbageCollectionRule(max_age=max_age) - rule2 = GarbageCollectionRule(max_num_versions=1) - - # Make a composite that matches anything older than 3 days **AND** - # with more than 1 version. - rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) - - # Make another composite that matches our previous intersection - # **OR** anything that has more than 3 versions. - rule4 = GarbageCollectionRule(max_num_versions=3) - rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) - ----- - -.. automodule:: gcloud_bigtable.column_family - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/data-api.rst b/docs/data-api.rst deleted file mode 100644 index fc41ad3..0000000 --- a/docs/data-api.rst +++ /dev/null @@ -1,327 +0,0 @@ -Data API -======== - -After creating a :class:`Table ` and some -column families, you are ready to store and retrieve data. - -Cells vs. Columns vs. Column Families -+++++++++++++++++++++++++++++++++++++ - -* As we saw before, a table can have many column families. -* As we'll see below, a table also has many rows (specified by row keys). -* Within a row, data is stored in a cell. A cell simply has a value (as - bytes) and a timestamp. The number of cells in each row can be - different, depending on what was stored in each row. -* Each cell lies in a column (**not** a column family). A column is really - just a more **specific** modifier within a column family. A column - can be present in every way, in only one or anywhere in between. -* Within a column family there can be many columns. For example within - the column family ``foo`` we could have columns ``bar`` and ``baz``. - These would typically be represented as ``foo:bar`` and ``foo:baz``. - -Modifying Data -++++++++++++++ - -Since data is stored in cells, which are stored in rows, the -:class:`Row ` class is the only class used to -modify (write, update, delete) data in a -:class:`Table `. - -Row Factory ------------ - -To create a :class:`Row ` object - -.. code:: python - - row = table.row(row_key) - -Unlike the previous string values we've used before, the row key must -be ``bytes``. - -Direct vs. Conditional vs. Append ---------------------------------- - -There are three ways to modify data in a table, described by the -`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API -methods. - -* The **direct** way is via `MutateRow`_ which involves simply - adding, overwriting or deleting cells. -* The **conditional** way is via `CheckAndMutateRow`_. This method - first checks if some filter is matched in a a given row, then - applies one of two sets of mutations, depending on if a match - occurred or not. -* The **append** way is via `ReadModifyWriteRow`_. This simply - appends (as bytes) or increments (as an integer) data in a presumed - existing cell in a row. - -Building Up Mutations ---------------------- - -In all three cases, a set of mutations (or two sets) are built up -on a :class:`Row ` before they are sent of -in a batch via :meth:`commit() `: - -.. code:: python - - row.commit() - -To send **append** mutations in batch, use -:meth:`commit_modifications() `: - -.. code:: python - - row.commit_modifications() - -We have a small set of methods on the :class:`Row ` -to build these mutations up. - -Direct Mutations ----------------- - -Direct mutations can be added via one of four methods - -* :meth:`set_cell() ` allows a - single value to be written to a column - - .. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp) - - If the ``timestamp`` is omitted, the current time on the Google Cloud - Bigtable server will be used when the cell is stored. - - The value can either by bytes or an integer (which will be converted to - bytes as an unsigned 64-bit integer). - -* :meth:`delete_cell() ` deletes - all cells (i.e. for all timestamps) in a given column - - .. code:: python - - row.delete_cell(column_family_id, column) - - Remember, this only happens in the ``row`` we are using. - - If we only want to delete cells from a limited range of time, a - :class:`TimestampRange ` can - be used - - .. code:: python - - row.delete_cell(column_family_id, column, - time_range=time_range) - -* :meth:`delete_cells() ` does - the same thing as :meth:`delete_cell() ` - but accepts a list of columns in a column family rather than a single one. - - .. code:: python - - row.delete_cells(column_family_id, [column1, column2], - time_range=time_range) - - In addition, if we want to delete cells from every column in a column family, - the special :attr:`ALL_COLUMNS ` value - can be used - - .. code:: python - - row.delete_cells(column_family_id, Row.ALL_COLUMNS, - time_range=time_range) - -* :meth:`delete() ` will delete the entire row - - .. code:: python - - row.delete() - -Conditional Mutations ---------------------- - -Making **conditional** conditional modifications is essentially identical -to **direct** modifications, but we need to specify a filter to match -against in the row: - -.. code:: python - - row = table.row(row_key, filter_=filter_val) - -See the :class:`Row ` class for more information -about acceptable values for ``filter_``. - -The only other difference from **direct** modifications are that each mutation -added must specify a ``state``: will the mutation be applied if the filter -matches or if it fails to match. - -For example - -.. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp, state=True) - -.. note:: - - If ``state`` is passed when no ``filter_`` is set on a - :class:`Row `, adding the mutation will fail. - Similarly, if no ``state`` is passed when a ``filter_`` has been set, - adding the mutation will fail. - -Append Mutations ----------------- - -Append mutations can be added via one of two methods - -* :meth:`append_cell_value ` appends - a bytes value to an existing cell: - - .. code:: python - - row.append_cell_value(column_family_id, column, bytes_value) - -* :meth:`increment_cell_value ` increments - an integer value in an existing cell: - - .. code:: python - - row.increment_cell_value(column_family_id, column, int_value) - - Since only bytes are stored in a cell, the current value is decoded as - an unsigned 64-bit integer before being incremented. (This happens on - the Google Cloud Bigtable server, not in the library.) - -Notice that no timestamp was specified. This is because **append** mutations -operate on the latest value of the specified column. - -If there are no cells in the specified column, then the empty string (bytes -case) or zero (integer case) are the assumed values. - -Starting Fresh --------------- - -If accumulated mutations need to be dropped, use -:meth:`clear_mutations() ` - -.. code:: python - - row.clear_mutations() - -To clear **append** mutations, use -:meth:`clear_modification_rules() ` - -.. code:: python - - row.clear_modification_rules() - -Reading Data -++++++++++++ - -Read Single Row from a Table ----------------------------- - -To make a `ReadRows`_ API request for a single row key, use -:meth:`Table.read_row() `: - -.. code:: python - - row_data = table.read_row(row_key) - -Rather than returning a :class:`Row `, this method -returns a :class:`PartialRowData ` -instance. This class is used for reading and parsing data rather than for -modifying data (as :class:`Row ` is). - -A filter can also be applied to the - -.. code:: python - - row_data = table.read_row(row_key, filter_=filter_val) - -The allowable ``filter_`` values are the same as those used for a -:class:`Row ` with **conditional** mutations. For -more information, see the -:meth:`Table.read_row() ` documentation. - -Stream Many Rows from a Table ------------------------------ - -To make a `ReadRows`_ API request for a stream of rows, use -:meth:`Table.read_rows() `: - -.. code:: python - - row_data = table.read_rows() - -Using gRPC over HTTP/2, a continual stream of responses will be delivered. -We have a custom -returns a :class:`PartialRowsData ` -class to allow consuming and parsing these streams as they come. - -In particular - -* :meth:`consume_next() ` - pulls the next result from the stream, parses it and stores it on the - :class:`PartialRowsData ` instance -* :meth:`consume_all() ` - pulls results from the stream until there are no more -* :meth:`cancel() ` closes - the stream - -See the :class:`PartialRowsData ` -documentation for more information. - -As with -:meth:`Table.read_row() `, an optional -``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` -can be supplied for the stream, a ``limit`` can be set and a boolean -``allow_row_interleaving`` can be specified to allow faster streamed results -at the potential cost of non-sequential reads. - -See the :meth:`Table.read_rows() ` -documentation for more information on the optional arguments. - -Sample Keys in a Table ----------------------- - -Make a `SampleRowKeys`_ API request with -:meth:`Table.sample_row_keys() `: - -.. code:: python - - keys_iterator = table.sample_row_keys() - -The returned row keys will delimit contiguous sections of the table of -approximately equal size, which can be used to break up the data for -distributed tasks like mapreduces. - -As with -:meth:`Table.read_rows() `, the -returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. - -The next key in the result can be accessed via - -.. code:: python - - next_key = keys_iterator.next() - -or all keys can be iterated over via - -.. code:: python - - for curr_key in keys_iterator: - do_something(curr_key) - -Just as with reading, the stream can be canceled: - -.. code:: python - - keys_iterator.cancel() - -.. _ReadRows: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L38 -.. _SampleRowKeys: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L44-L46 -.. _MutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L50-L52 -.. _CheckAndMutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L55-L57 -.. _ReadModifyWriteRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L63-L65 diff --git a/docs/happybase-batch.rst b/docs/happybase-batch.rst deleted file mode 100644 index 5c386e8..0000000 --- a/docs/happybase-batch.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Batch -~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.batch - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-connection.rst b/docs/happybase-connection.rst deleted file mode 100644 index 7d60940..0000000 --- a/docs/happybase-connection.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Connection -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.connection - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-package.rst b/docs/happybase-package.rst deleted file mode 100644 index 717d549..0000000 --- a/docs/happybase-package.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Package -~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.__init__ - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-pool.rst b/docs/happybase-pool.rst deleted file mode 100644 index 0b36ae1..0000000 --- a/docs/happybase-pool.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Connection Pool -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.pool - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-table.rst b/docs/happybase-table.rst deleted file mode 100644 index 0809e52..0000000 --- a/docs/happybase-table.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Table -~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.table - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index 77da93f..6c72fe8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,82 +1,8 @@ -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Getting Started - - client-intro - client - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Cluster Admin - - cluster-api - cluster - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Table Admin - - table-api - table - column-family - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Data API - - data-api - row - row-data - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: HappyBase Compatibility Sub-Package - - happybase-package - happybase-table - happybase-connection - happybase-batch - happybase-pool - Google Cloud Bigtable: Python ============================= -This library is an alpha implementation of `Google Cloud Bigtable`_ -and is closely related to `gcloud-python`_. - -API requests are sent to the Google Cloud Bigtable API via RPC over HTTP/2. -In order to support this, we'll rely on `gRPC`_. We are working with the gRPC -team to rapidly make the install story more user-friendly. - -Get started by learning about the -:class:`Client ` on the `Base for Everything`_ -page. If you have install questions, check out the project's `README`_. - -In the hierarchy of API concepts - -* a :class:`Client ` owns a - :class:`Cluster ` -* a :class:`Cluster ` owns a - :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily ` -* a :class:`Table ` owns a - :class:`Row ` - (and all the cells in the row) - -Indices and tables -~~~~~~~~~~~~~~~~~~ - -* :ref:`genindex` -* :ref:`modindex` +This library was an alpha implementation of `Google Cloud Bigtable`_. +The project has been moved into `gcloud-python`_. .. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ -.. _gcloud-python: http://gcloud-python.readthedocs.org/en/latest/ -.. _gRPC: http://www.grpc.io/ -.. _README: https://github.com/dhermes/gcloud-python-bigtable/blob/master/README.md -.. _Base for Everything: client-intro.html +.. _gcloud-python: http://gcloud-python.readthedocs.io/en/stable/bigtable-usage.html diff --git a/docs/row-data.rst b/docs/row-data.rst deleted file mode 100644 index 7a4c235..0000000 --- a/docs/row-data.rst +++ /dev/null @@ -1,7 +0,0 @@ -Row Data -~~~~~~~~ - -.. automodule:: gcloud_bigtable.row_data - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/row.rst b/docs/row.rst deleted file mode 100644 index 4603702..0000000 --- a/docs/row.rst +++ /dev/null @@ -1,7 +0,0 @@ -Row -~~~ - -.. automodule:: gcloud_bigtable.row - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/table-api.rst b/docs/table-api.rst deleted file mode 100644 index f90719c..0000000 --- a/docs/table-api.rst +++ /dev/null @@ -1,172 +0,0 @@ -Table Admin API -=============== - -After creating a :class:`Cluster `, you can -interact with individual tables, groups of tables or column families within -a table. - -List Tables ------------ - -If you want a comprehensive list of all existing tables in a cluster, make a -`ListTables`_ API request with -:meth:`Cluster.list_tables() `: - -.. code:: python - - tables = cluster.list_tables() - -Table Factory -------------- - -To create a :class:`Table ` object: - -.. code:: python - - table = cluster.table(table_id) - -Even if this :class:`Table ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`ColumnFamily ` -or :class:`Row `. - -Create a new Table ------------------- - -After creating the table object, make a `CreateTable`_ API request -with :meth:`create() `: - -.. code:: python - - table.create() - -If you would to initially split the table into several tablets (Tablets are -similar to HBase regions): - -.. code:: python - - table.create(initial_split_keys=['s1', 's2']) - -Delete an existing Table ------------------------- - -Make a `DeleteTable`_ API request with -:meth:`delete() `: - -.. code:: python - - table.delete() - -Rename an existing Table ------------------------- - -Though the `RenameTable`_ API request is listed in the service -definition, requests to that method return:: - - BigtableTableService.RenameTable is not yet implemented - -We have implemented :meth:`rename() ` -but it will not work unless the backend supports the method. - -List Column Families in a Table -------------------------------- - -Though there is no **official** method for retrieving `column families`_ -associated with a table, the `GetTable`_ API method returns a -table object with the names of the column families. - -To retrieve the list of column families use -:meth:`list_column_families() `: - -.. code:: python - - column_families = table.list_column_families() - -.. note:: - - Unfortunately the garbage collection rules used to create each column family - are not returned in the `GetTable`_ response. - -Column Family Factory ---------------------- - -To create a -:class:`ColumnFamily ` object: - -.. code:: python - - column_family = table.column_family(column_family_id) - -There is no real reason to use this factory unless you intend to -create or delete a column family. - -In addition, you can specify an optional ``gc_rule`` (a -:class:`GarbageCollectionRule ` -or similar): - -.. code:: python - - column_family = table.column_family(column_family_id, - gc_rule=gc_rule) - -This rule helps the backend determine when and how to clean up old cells -in the column family. - -See the `Column Families doc`_ for more information about -:class:`GarbageCollectionRule ` -and related classes. - -Create a new Column Family --------------------------- - -After creating the column family object, make a `CreateColumnFamily`_ API -request with -:meth:`ColumnFamily.create() ` - -.. code:: python - - column_family.create() - -Delete an existing Column Family --------------------------------- - -Make a `DeleteColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.delete() - -Update an existing Column Family --------------------------------- - -Though the `UpdateColumnFamily`_ API request is listed in the service -definition, requests to that method return:: - - BigtableTableService.UpdateColumnFamily is not yet implemented - -We have implemented -:meth:`ColumnFamily.update() ` -but it will not work unless the backend supports the method. - -Next Step ---------- - -Now we go down the final step of the hierarchy from -:class:`Table ` to -:class:`Row ` as well as streaming -data directly via a :class:`Table `. - -Head next to learn about the `Data API`_. - -.. _ListTables: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L40-L42 -.. _CreateTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L35-L37 -.. _DeleteTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L50-L52 -.. _RenameTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L58-L58 -.. _GetTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L45-L47 -.. _CreateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L61-L63 -.. _UpdateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L66-L68 -.. _DeleteColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L71-L73 -.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers -.. _Column Families doc: column-family.html -.. _Data API: data-api.html diff --git a/docs/table.rst b/docs/table.rst deleted file mode 100644 index affb0f1..0000000 --- a/docs/table.rst +++ /dev/null @@ -1,7 +0,0 @@ -Table -~~~~~ - -.. automodule:: gcloud_bigtable.table - :members: - :undoc-members: - :show-inheritance: diff --git a/gcloud_bigtable/__init__.py b/gcloud_bigtable/__init__.py deleted file mode 100644 index 54bb2ec..0000000 --- a/gcloud_bigtable/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable API package.""" - - -from gcloud_bigtable.client import Client diff --git a/gcloud_bigtable/_generated/__init__.py b/gcloud_bigtable/_generated/__init__.py deleted file mode 100644 index ad35adc..0000000 --- a/gcloud_bigtable/_generated/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generated protobuf modules for Google Cloud Bigtable API.""" diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_data.proto b/gcloud_bigtable/_generated/_bigtable_cluster_data.proto deleted file mode 100644 index c0f8a93..0000000 --- a/gcloud_bigtable/_generated/_bigtable_cluster_data.proto +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.cluster.v1; - -import "google/api/annotations.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/timestamp.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableClusterDataProto"; -option java_package = "com.google.bigtable.admin.cluster.v1"; - - -// A physical location in which a particular project can allocate Cloud BigTable -// resources. -message Zone { - // Possible states of a zone. - enum Status { - // The state of the zone is unknown or unspecified. - UNKNOWN = 0; - - // The zone is in a good state. - OK = 1; - - // The zone is down for planned maintenance. - PLANNED_MAINTENANCE = 2; - - // The zone is down for emergency or unplanned maintenance. - EMERGENCY_MAINENANCE = 3; - } - - // A permanent unique identifier for the zone. - // Values are of the form projects//zones/[a-z][-a-z0-9]* - string name = 1; - - // The name of this zone as it appears in UIs. - string display_name = 2; - - // The current state of this zone. - Status status = 3; -} - -// An isolated set of Cloud BigTable resources on which tables can be hosted. -message Cluster { - // A permanent unique identifier for the cluster. For technical reasons, the - // zone in which the cluster resides is included here. - // Values are of the form - // projects//zones//clusters/[a-z][-a-z0-9]* - string name = 1; - - // The operation currently running on the cluster, if any. - // This cannot be set directly, only through CreateCluster, UpdateCluster, - // or UndeleteCluster. Calls to these methods will be rejected if - // "current_operation" is already set. - google.longrunning.Operation current_operation = 3; - - // The descriptive name for this cluster as it appears in UIs. - // Must be unique per zone. - string display_name = 4; - - // The number of serve nodes allocated to this cluster. - int32 serve_nodes = 5; - - // What storage type to use for tables in this cluster. Only configurable at - // cluster creation time. If unspecified, STORAGE_SSD will be used. - StorageType default_storage_type = 8; -} - -enum StorageType { - // The storage type used is unspecified. - STORAGE_UNSPECIFIED = 0; - - // Data will be stored in SSD, providing low and consistent latencies. - STORAGE_SSD = 1; - - // Data will be stored in HDD, providing high and less predictable - // latencies. - STORAGE_HDD = 2; -} diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_service.proto b/gcloud_bigtable/_generated/_bigtable_cluster_service.proto deleted file mode 100644 index e65bca4..0000000 --- a/gcloud_bigtable/_generated/_bigtable_cluster_service.proto +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.cluster.v1; - -import "google/api/annotations.proto"; -import "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto"; -import "google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/empty.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableClusterServicesProto"; -option java_package = "com.google.bigtable.admin.cluster.v1"; - - -// Service for managing zonal Cloud Bigtable resources. -service BigtableClusterService { - // Lists the supported zones for the given project. - rpc ListZones(ListZonesRequest) returns (ListZonesResponse) { - option (google.api.http) = { get: "/v1/{name=projects/*}/zones" }; - } - - // Gets information about a particular cluster. - rpc GetCluster(GetClusterRequest) returns (Cluster) { - option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*}" }; - } - - // Lists all clusters in the given project, along with any zones for which - // cluster information could not be retrieved. - rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) { - option (google.api.http) = { get: "/v1/{name=projects/*}/aggregated/clusters" }; - } - - // Creates a cluster and begins preparing it to begin serving. The returned - // cluster embeds as its "current_operation" a long-running operation which - // can be used to track the progress of turning up the new cluster. - // Immediately upon completion of this request: - // * The cluster will be readable via the API, with all requested attributes - // but no allocated resources. - // Until completion of the embedded operation: - // * Cancelling the operation will render the cluster immediately unreadable - // via the API. - // * All other attempts to modify or delete the cluster will be rejected. - // Upon completion of the embedded operation: - // * Billing for all successfully-allocated resources will begin (some types - // may have lower than the requested levels). - // * New tables can be created in the cluster. - // * The cluster's allocated resource levels will be readable via the API. - // The embedded operation's "metadata" field type is - // [CreateClusterMetadata][google.bigtable.admin.cluster.v1.CreateClusterMetadata] The embedded operation's "response" field type is - // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. - rpc CreateCluster(CreateClusterRequest) returns (Cluster) { - option (google.api.http) = { post: "/v1/{name=projects/*/zones/*}/clusters" body: "*" }; - } - - // Updates a cluster, and begins allocating or releasing resources as - // requested. The returned cluster embeds as its "current_operation" a - // long-running operation which can be used to track the progress of updating - // the cluster. - // Immediately upon completion of this request: - // * For resource types where a decrease in the cluster's allocation has been - // requested, billing will be based on the newly-requested level. - // Until completion of the embedded operation: - // * Cancelling the operation will set its metadata's "cancelled_at_time", - // and begin restoring resources to their pre-request values. The operation - // is guaranteed to succeed at undoing all resource changes, after which - // point it will terminate with a CANCELLED status. - // * All other attempts to modify or delete the cluster will be rejected. - // * Reading the cluster via the API will continue to give the pre-request - // resource levels. - // Upon completion of the embedded operation: - // * Billing will begin for all successfully-allocated resources (some types - // may have lower than the requested levels). - // * All newly-reserved resources will be available for serving the cluster's - // tables. - // * The cluster's new resource levels will be readable via the API. - // [UpdateClusterMetadata][google.bigtable.admin.cluster.v1.UpdateClusterMetadata] The embedded operation's "response" field type is - // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. - rpc UpdateCluster(Cluster) returns (Cluster) { - option (google.api.http) = { put: "/v1/{name=projects/*/zones/*/clusters/*}" body: "*" }; - } - - // Marks a cluster and all of its tables for permanent deletion in 7 days. - // Immediately upon completion of the request: - // * Billing will cease for all of the cluster's reserved resources. - // * The cluster's "delete_time" field will be set 7 days in the future. - // Soon afterward: - // * All tables within the cluster will become unavailable. - // Prior to the cluster's "delete_time": - // * The cluster can be recovered with a call to UndeleteCluster. - // * All other attempts to modify or delete the cluster will be rejected. - // At the cluster's "delete_time": - // * The cluster and *all of its tables* will immediately and irrevocably - // disappear from the API, and their data will be permanently deleted. - rpc DeleteCluster(DeleteClusterRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*}" }; - } - - // Cancels the scheduled deletion of an cluster and begins preparing it to - // resume serving. The returned operation will also be embedded as the - // cluster's "current_operation". - // Immediately upon completion of this request: - // * The cluster's "delete_time" field will be unset, protecting it from - // automatic deletion. - // Until completion of the returned operation: - // * The operation cannot be cancelled. - // Upon completion of the returned operation: - // * Billing for the cluster's resources will resume. - // * All tables within the cluster will be available. - // [UndeleteClusterMetadata][google.bigtable.admin.cluster.v1.UndeleteClusterMetadata] The embedded operation's "response" field type is - // [Cluster][google.bigtable.admin.cluster.v1.Cluster], if successful. - rpc UndeleteCluster(UndeleteClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*}:undelete" body: "null" }; - } -} diff --git a/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto deleted file mode 100644 index 3291969..0000000 --- a/gcloud_bigtable/_generated/_bigtable_cluster_service_messages.proto +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.cluster.v1; - -import "google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto"; -import "google/protobuf/timestamp.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableClusterServiceMessagesProto"; -option java_package = "com.google.bigtable.admin.cluster.v1"; - - -// Request message for BigtableClusterService.ListZones. -message ListZonesRequest { - // The unique name of the project for which a list of supported zones is - // requested. - // Values are of the form projects/ - string name = 1; -} - -// Response message for BigtableClusterService.ListZones. -message ListZonesResponse { - // The list of requested zones. - repeated Zone zones = 1; -} - -// Request message for BigtableClusterService.GetCluster. -message GetClusterRequest { - // The unique name of the requested cluster. - // Values are of the form projects//zones//clusters/ - string name = 1; -} - -// Request message for BigtableClusterService.ListClusters. -message ListClustersRequest { - // The unique name of the project for which a list of clusters is requested. - // Values are of the form projects/ - string name = 1; -} - -// Response message for BigtableClusterService.ListClusters. -message ListClustersResponse { - // The list of requested Clusters. - repeated Cluster clusters = 1; - - // The zones for which clusters could not be retrieved. - repeated Zone failed_zones = 2; -} - -// Request message for BigtableClusterService.CreateCluster. -message CreateClusterRequest { - // The unique name of the zone in which to create the cluster. - // Values are of the form projects//zones/ - string name = 1; - - // The id to be used when referring to the new cluster within its zone, - // e.g. just the "test-cluster" section of the full name - // "projects//zones//clusters/test-cluster". - string cluster_id = 2; - - // The cluster to create. - // The "name", "delete_time", and "current_operation" fields must be left - // blank. - Cluster cluster = 3; -} - -// Metadata type for the operation returned by -// BigtableClusterService.CreateCluster. -message CreateClusterMetadata { - // The request which prompted the creation of this operation. - CreateClusterRequest original_request = 1; - - // The time at which original_request was received. - google.protobuf.Timestamp request_time = 2; - - // The time at which this operation failed or was completed successfully. - google.protobuf.Timestamp finish_time = 3; -} - -// Metadata type for the operation returned by -// BigtableClusterService.UpdateCluster. -message UpdateClusterMetadata { - // The request which prompted the creation of this operation. - Cluster original_request = 1; - - // The time at which original_request was received. - google.protobuf.Timestamp request_time = 2; - - // The time at which this operation was cancelled. If set, this operation is - // in the process of undoing itself (which is guaranteed to succeed) and - // cannot be cancelled again. - google.protobuf.Timestamp cancel_time = 3; - - // The time at which this operation failed or was completed successfully. - google.protobuf.Timestamp finish_time = 4; -} - -// Request message for BigtableClusterService.DeleteCluster. -message DeleteClusterRequest { - // The unique name of the cluster to be deleted. - // Values are of the form projects//zones//clusters/ - string name = 1; -} - -// Request message for BigtableClusterService.UndeleteCluster. -message UndeleteClusterRequest { - // The unique name of the cluster to be un-deleted. - // Values are of the form projects//zones//clusters/ - string name = 1; -} - -// Metadata type for the operation returned by -// BigtableClusterService.UndeleteCluster. -message UndeleteClusterMetadata { - // The time at which the original request was received. - google.protobuf.Timestamp request_time = 1; - - // The time at which this operation failed or was completed successfully. - google.protobuf.Timestamp finish_time = 2; -} diff --git a/gcloud_bigtable/_generated/_bigtable_data.proto b/gcloud_bigtable/_generated/_bigtable_data.proto deleted file mode 100644 index 290eb91..0000000 --- a/gcloud_bigtable/_generated/_bigtable_data.proto +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.v1; - -option java_multiple_files = true; -option java_outer_classname = "BigtableDataProto"; -option java_package = "com.google.bigtable.v1"; - - -// Specifies the complete (requested) contents of a single row of a table. -// Rows which exceed 256MiB in size cannot be read in full. -message Row { - // The unique key which identifies this row within its table. This is the same - // key that's used to identify the row in, for example, a MutateRowRequest. - // May contain any non-empty byte string up to 4KiB in length. - bytes key = 1; - - // May be empty, but only if the entire row is empty. - // The mutual ordering of column families is not specified. - repeated Family families = 2; -} - -// Specifies (some of) the contents of a single row/column family of a table. -message Family { - // The unique key which identifies this family within its row. This is the - // same key that's used to identify the family in, for example, a RowFilter - // which sets its "family_name_regex_filter" field. - // Must match [-_.a-zA-Z0-9]+, except that AggregatingRowProcessors may - // produce cells in a sentinel family with an empty name. - // Must be no greater than 64 characters in length. - string name = 1; - - // Must not be empty. Sorted in order of increasing "qualifier". - repeated Column columns = 2; -} - -// Specifies (some of) the contents of a single row/column of a table. -message Column { - // The unique key which identifies this column within its family. This is the - // same key that's used to identify the column in, for example, a RowFilter - // which sets its "column_qualifier_regex_filter" field. - // May contain any byte string, including the empty string, up to 16kiB in - // length. - bytes qualifier = 1; - - // Must not be empty. Sorted in order of decreasing "timestamp_micros". - repeated Cell cells = 2; -} - -// Specifies (some of) the contents of a single row/column/timestamp of a table. -message Cell { - // The cell's stored timestamp, which also uniquely identifies it within - // its column. - // Values are always expressed in microseconds, but individual tables may set - // a coarser "granularity" to further restrict the allowed values. For - // example, a table which specifies millisecond granularity will only allow - // values of "timestamp_micros" which are multiples of 1000. - int64 timestamp_micros = 1; - - // The value stored in the cell. - // May contain any byte string, including the empty string, up to 100MiB in - // length. - bytes value = 2; - - // Labels applied to the cell by a [RowFilter][google.bigtable.v1.RowFilter]. - repeated string labels = 3; -} - -// Specifies a contiguous range of rows. -message RowRange { - // Inclusive lower bound. If left empty, interpreted as the empty string. - bytes start_key = 2; - - // Exclusive upper bound. If left empty, interpreted as infinity. - bytes end_key = 3; -} - -// Specifies a non-contiguous set of rows. -message RowSet { - // Single rows included in the set. - repeated bytes row_keys = 1; - - // Contiguous row ranges included in the set. - repeated RowRange row_ranges = 2; -} - -// Specifies a contiguous range of columns within a single column family. -// The range spans from : to -// :, where both bounds can be either inclusive or -// exclusive. -message ColumnRange { - // The name of the column family within which this range falls. - string family_name = 1; - - // The column qualifier at which to start the range (within 'column_family'). - // If neither field is set, interpreted as the empty string, inclusive. - oneof start_qualifier { - // Used when giving an inclusive lower bound for the range. - bytes start_qualifier_inclusive = 2; - - // Used when giving an exclusive lower bound for the range. - bytes start_qualifier_exclusive = 3; - } - - // The column qualifier at which to end the range (within 'column_family'). - // If neither field is set, interpreted as the infinite string, exclusive. - oneof end_qualifier { - // Used when giving an inclusive upper bound for the range. - bytes end_qualifier_inclusive = 4; - - // Used when giving an exclusive upper bound for the range. - bytes end_qualifier_exclusive = 5; - } -} - -// Specified a contiguous range of microsecond timestamps. -message TimestampRange { - // Inclusive lower bound. If left empty, interpreted as 0. - int64 start_timestamp_micros = 1; - - // Exclusive upper bound. If left empty, interpreted as infinity. - int64 end_timestamp_micros = 2; -} - -// Specifies a contiguous range of raw byte values. -message ValueRange { - // The value at which to start the range. - // If neither field is set, interpreted as the empty string, inclusive. - oneof start_value { - // Used when giving an inclusive lower bound for the range. - bytes start_value_inclusive = 1; - - // Used when giving an exclusive lower bound for the range. - bytes start_value_exclusive = 2; - } - - // The value at which to end the range. - // If neither field is set, interpreted as the infinite string, exclusive. - oneof end_value { - // Used when giving an inclusive upper bound for the range. - bytes end_value_inclusive = 3; - - // Used when giving an exclusive upper bound for the range. - bytes end_value_exclusive = 4; - } -} - -// Takes a row as input and produces an alternate view of the row based on -// specified rules. For example, a RowFilter might trim down a row to include -// just the cells from columns matching a given regular expression, or might -// return all the cells of a row but not their values. More complicated filters -// can be composed out of these components to express requests such as, "within -// every column of a particular family, give just the two most recent cells -// which are older than timestamp X." -// -// There are two broad categories of RowFilters (true filters and transformers), -// as well as two ways to compose simple filters into more complex ones -// (chains and interleaves). They work as follows: -// -// * True filters alter the input row by excluding some of its cells wholesale -// from the output row. An example of a true filter is the "value_regex_filter", -// which excludes cells whose values don't match the specified pattern. All -// regex true filters use RE2 syntax (https://github.com/google/re2/wiki/Syntax) -// in raw byte mode (RE2::Latin1), and are evaluated as full matches. An -// important point to keep in mind is that RE2(.) is equivalent by default to -// RE2([^\n]), meaning that it does not match newlines. When attempting to match -// an arbitrary byte, you should therefore use the escape sequence '\C', which -// may need to be further escaped as '\\C' in your client language. -// -// * Transformers alter the input row by changing the values of some of its -// cells in the output, without excluding them completely. Currently, the only -// supported transformer is the "strip_value_transformer", which replaces every -// cell's value with the empty string. -// -// * Chains and interleaves are described in more detail in the -// RowFilter.Chain and RowFilter.Interleave documentation. -// -// The total serialized size of a RowFilter message must not -// exceed 4096 bytes, and RowFilters may not be nested within each other -// (in Chains or Interleaves) to a depth of more than 20. -message RowFilter { - // A RowFilter which sends rows through several RowFilters in sequence. - message Chain { - // The elements of "filters" are chained together to process the input row: - // in row -> f(0) -> intermediate row -> f(1) -> ... -> f(N) -> out row - // The full chain is executed atomically. - repeated RowFilter filters = 1; - } - - // A RowFilter which sends each row to each of several component - // RowFilters and interleaves the results. - message Interleave { - // The elements of "filters" all process a copy of the input row, and the - // results are pooled, sorted, and combined into a single output row. - // If multiple cells are produced with the same column and timestamp, - // they will all appear in the output row in an unspecified mutual order. - // Consider the following example, with three filters: - // - // input row - // | - // ----------------------------------------------------- - // | | | - // f(0) f(1) f(2) - // | | | - // 1: foo,bar,10,x foo,bar,10,z far,bar,7,a - // 2: foo,blah,11,z far,blah,5,x far,blah,5,x - // | | | - // ----------------------------------------------------- - // | - // 1: foo,bar,10,z // could have switched with #2 - // 2: foo,bar,10,x // could have switched with #1 - // 3: foo,blah,11,z - // 4: far,bar,7,a - // 5: far,blah,5,x // identical to #6 - // 6: far,blah,5,x // identical to #5 - // All interleaved filters are executed atomically. - repeated RowFilter filters = 1; - } - - // A RowFilter which evaluates one of two possible RowFilters, depending on - // whether or not a predicate RowFilter outputs any cells from the input row. - // - // IMPORTANT NOTE: The predicate filter does not execute atomically with the - // true and false filters, which may lead to inconsistent or unexpected - // results. Additionally, Condition filters have poor performance, especially - // when filters are set for the false condition. - message Condition { - // If "predicate_filter" outputs any cells, then "true_filter" will be - // evaluated on the input row. Otherwise, "false_filter" will be evaluated. - RowFilter predicate_filter = 1; - - // The filter to apply to the input row if "predicate_filter" returns any - // results. If not provided, no results will be returned in the true case. - RowFilter true_filter = 2; - - // The filter to apply to the input row if "predicate_filter" does not - // return any results. If not provided, no results will be returned in the - // false case. - RowFilter false_filter = 3; - } - - // Which of the possible RowFilter types to apply. If none are set, this - // RowFilter returns all cells in the input row. - oneof filter { - // Applies several RowFilters to the data in sequence, progressively - // narrowing the results. - Chain chain = 1; - - // Applies several RowFilters to the data in parallel and combines the - // results. - Interleave interleave = 2; - - // Applies one of two possible RowFilters to the data based on the output of - // a predicate RowFilter. - Condition condition = 3; - - // ADVANCED USE ONLY. - // Hook for introspection into the RowFilter. Outputs all cells directly to - // the output of the read rather than to any parent filter. Consider the - // following example: - // - // Chain( - // FamilyRegex("A"), - // Interleave( - // All(), - // Chain(Label("foo"), Sink()) - // ), - // QualifierRegex("B") - // ) - // - // A,A,1,w - // A,B,2,x - // B,B,4,z - // | - // FamilyRegex("A") - // | - // A,A,1,w - // A,B,2,x - // | - // +------------+-------------+ - // | | - // All() Label(foo) - // | | - // A,A,1,w A,A,1,w,labels:[foo] - // A,B,2,x A,B,2,x,labels:[foo] - // | | - // | Sink() --------------+ - // | | | - // +------------+ x------+ A,A,1,w,labels:[foo] - // | A,B,2,x,labels:[foo] - // A,A,1,w | - // A,B,2,x | - // | | - // QualifierRegex("B") | - // | | - // A,B,2,x | - // | | - // +--------------------------------+ - // | - // A,A,1,w,labels:[foo] - // A,B,2,x,labels:[foo] // could be switched - // A,B,2,x // could be switched - // - // Despite being excluded by the qualifier filter, a copy of every cell - // that reaches the sink is present in the final result. - // - // As with an [Interleave][google.bigtable.v1.RowFilter.Interleave], - // duplicate cells are possible, and appear in an unspecified mutual order. - // In this case we have a duplicate with column "A:B" and timestamp 2, - // because one copy passed through the all filter while the other was - // passed through the label and sink. Note that one copy has label "foo", - // while the other does not. - // - // Cannot be used within the `predicate_filter`, `true_filter`, or - // `false_filter` of a [Condition][google.bigtable.v1.RowFilter.Condition]. - bool sink = 16; - - // Matches all cells, regardless of input. Functionally equivalent to - // leaving `filter` unset, but included for completeness. - bool pass_all_filter = 17; - - // Does not match any cells, regardless of input. Useful for temporarily - // disabling just part of a filter. - bool block_all_filter = 18; - - // Matches only cells from rows whose keys satisfy the given RE2 regex. In - // other words, passes through the entire row when the key matches, and - // otherwise produces an empty row. - // Note that, since row keys can contain arbitrary bytes, the '\C' escape - // sequence must be used if a true wildcard is desired. The '.' character - // will not match the new line character '\n', which may be present in a - // binary key. - bytes row_key_regex_filter = 4; - - // Matches all cells from a row with probability p, and matches no cells - // from the row with probability 1-p. - double row_sample_filter = 14; - - // Matches only cells from columns whose families satisfy the given RE2 - // regex. For technical reasons, the regex must not contain the ':' - // character, even if it is not being used as a literal. - // Note that, since column families cannot contain the new line character - // '\n', it is sufficient to use '.' as a full wildcard when matching - // column family names. - string family_name_regex_filter = 5; - - // Matches only cells from columns whose qualifiers satisfy the given RE2 - // regex. - // Note that, since column qualifiers can contain arbitrary bytes, the '\C' - // escape sequence must be used if a true wildcard is desired. The '.' - // character will not match the new line character '\n', which may be - // present in a binary qualifier. - bytes column_qualifier_regex_filter = 6; - - // Matches only cells from columns within the given range. - ColumnRange column_range_filter = 7; - - // Matches only cells with timestamps within the given range. - TimestampRange timestamp_range_filter = 8; - - // Matches only cells with values that satisfy the given regular expression. - // Note that, since cell values can contain arbitrary bytes, the '\C' escape - // sequence must be used if a true wildcard is desired. The '.' character - // will not match the new line character '\n', which may be present in a - // binary value. - bytes value_regex_filter = 9; - - // Matches only cells with values that fall within the given range. - ValueRange value_range_filter = 15; - - // Skips the first N cells of each row, matching all subsequent cells. - // If duplicate cells are present, as is possible when using an Interleave, - // each copy of the cell is counted separately. - int32 cells_per_row_offset_filter = 10; - - // Matches only the first N cells of each row. - // If duplicate cells are present, as is possible when using an Interleave, - // each copy of the cell is counted separately. - int32 cells_per_row_limit_filter = 11; - - // Matches only the most recent N cells within each column. For example, - // if N=2, this filter would match column "foo:bar" at timestamps 10 and 9, - // skip all earlier cells in "foo:bar", and then begin matching again in - // column "foo:bar2". - // If duplicate cells are present, as is possible when using an Interleave, - // each copy of the cell is counted separately. - int32 cells_per_column_limit_filter = 12; - - // Replaces each cell's value with the empty string. - bool strip_value_transformer = 13; - - // Applies the given label to all cells in the output row. This allows - // the client to determine which results were produced from which part of - // the filter. - // - // Values must be at most 15 characters in length, and match the RE2 - // pattern [a-z0-9\\-]+ - // - // Due to a technical limitation, it is not currently possible to apply - // multiple labels to a cell. As a result, a Chain may have no more than - // one sub-filter which contains a apply_label_transformer. It is okay for - // an Interleave to contain multiple apply_label_transformers, as they will - // be applied to separate copies of the input. This may be relaxed in the - // future. - string apply_label_transformer = 19; - } -} - -// Specifies a particular change to be made to the contents of a row. -message Mutation { - // A Mutation which sets the value of the specified cell. - message SetCell { - // The name of the family into which new data should be written. - // Must match [-_.a-zA-Z0-9]+ - string family_name = 1; - - // The qualifier of the column into which new data should be written. - // Can be any byte string, including the empty string. - bytes column_qualifier = 2; - - // The timestamp of the cell into which new data should be written. - // Use -1 for current Bigtable server time. - // Otherwise, the client should set this value itself, noting that the - // default value is a timestamp of zero if the field is left unspecified. - // Values must match the "granularity" of the table (e.g. micros, millis). - int64 timestamp_micros = 3; - - // The value to be written into the specified cell. - bytes value = 4; - } - - // A Mutation which deletes cells from the specified column, optionally - // restricting the deletions to a given timestamp range. - message DeleteFromColumn { - // The name of the family from which cells should be deleted. - // Must match [-_.a-zA-Z0-9]+ - string family_name = 1; - - // The qualifier of the column from which cells should be deleted. - // Can be any byte string, including the empty string. - bytes column_qualifier = 2; - - // The range of timestamps within which cells should be deleted. - TimestampRange time_range = 3; - } - - // A Mutation which deletes all cells from the specified column family. - message DeleteFromFamily { - // The name of the family from which cells should be deleted. - // Must match [-_.a-zA-Z0-9]+ - string family_name = 1; - } - - // A Mutation which deletes all cells from the containing row. - message DeleteFromRow { - - } - - // Which of the possible Mutation types to apply. - oneof mutation { - // Set a cell's value. - SetCell set_cell = 1; - - // Deletes cells from a column. - DeleteFromColumn delete_from_column = 2; - - // Deletes cells from a column family. - DeleteFromFamily delete_from_family = 3; - - // Deletes cells from the entire row. - DeleteFromRow delete_from_row = 4; - } -} - -// Specifies an atomic read/modify/write operation on the latest value of the -// specified column. -message ReadModifyWriteRule { - // The name of the family to which the read/modify/write should be applied. - // Must match [-_.a-zA-Z0-9]+ - string family_name = 1; - - // The qualifier of the column to which the read/modify/write should be - // applied. - // Can be any byte string, including the empty string. - bytes column_qualifier = 2; - - // The rule used to determine the column's new latest value from its current - // latest value. - oneof rule { - // Rule specifying that "append_value" be appended to the existing value. - // If the targeted cell is unset, it will be treated as containing the - // empty string. - bytes append_value = 3; - - // Rule specifying that "increment_amount" be added to the existing value. - // If the targeted cell is unset, it will be treated as containing a zero. - // Otherwise, the targeted cell must contain an 8-byte value (interpreted - // as a 64-bit big-endian signed integer), or the entire request will fail. - int64 increment_amount = 4; - } -} diff --git a/gcloud_bigtable/_generated/_bigtable_service.proto b/gcloud_bigtable/_generated/_bigtable_service.proto deleted file mode 100644 index f1a83d3..0000000 --- a/gcloud_bigtable/_generated/_bigtable_service.proto +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.v1; - -import "google/api/annotations.proto"; -import "google/bigtable/v1/bigtable_data.proto"; -import "google/bigtable/v1/bigtable_service_messages.proto"; -import "google/protobuf/empty.proto"; - -option java_generic_services = true; -option java_multiple_files = true; -option java_outer_classname = "BigtableServicesProto"; -option java_package = "com.google.bigtable.v1"; - - -// Service for reading from and writing to existing Bigtables. -service BigtableService { - // Streams back the contents of all requested rows, optionally applying - // the same Reader filter to each. Depending on their size, rows may be - // broken up across multiple responses, but atomicity of each row will still - // be preserved. - rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) { - option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read" body: "*" }; - } - - // Returns a sample of row keys in the table. The returned row keys will - // delimit contiguous sections of the table of approximately equal size, - // which can be used to break up the data for distributed tasks like - // mapreduces. - rpc SampleRowKeys(SampleRowKeysRequest) returns (stream SampleRowKeysResponse) { - option (google.api.http) = { get: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys" }; - } - - // Mutates a row atomically. Cells already present in the row are left - // unchanged unless explicitly changed by 'mutation'. - rpc MutateRow(MutateRowRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate" body: "*" }; - } - - // Mutates multiple rows in a batch. Each individual row is mutated - // atomically as in MutateRow, but the entire batch is not executed - // atomically. - rpc MutateRows(MutateRowsRequest) returns (MutateRowsResponse) { - option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}:mutateRows" body: "*" }; - } - - // Mutates a row atomically based on the output of a predicate Reader filter. - rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) { - option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate" body: "*" }; - } - - // Modifies a row atomically, reading the latest existing timestamp/value from - // the specified columns and writing a new value at - // max(existing timestamp, current server time) based on pre-defined - // read/modify/write rules. Returns the new contents of all modified cells. - rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (Row) { - option (google.api.http) = { post: "/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite" body: "*" }; - } -} diff --git a/gcloud_bigtable/_generated/_bigtable_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_service_messages.proto deleted file mode 100644 index 1479fb6..0000000 --- a/gcloud_bigtable/_generated/_bigtable_service_messages.proto +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.v1; - -import "google/bigtable/v1/bigtable_data.proto"; -import "google/rpc/status.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableServiceMessagesProto"; -option java_package = "com.google.bigtable.v1"; - - -// Request message for BigtableServer.ReadRows. -message ReadRowsRequest { - // The unique name of the table from which to read. - string table_name = 1; - - // If neither row_key nor row_range is set, reads from all rows. - oneof target { - // The key of a single row from which to read. - bytes row_key = 2; - - // A range of rows from which to read. - RowRange row_range = 3; - - // A set of rows from which to read. Entries need not be in order, and will - // be deduplicated before reading. - // The total serialized size of the set must not exceed 1MB. - RowSet row_set = 8; - } - - // The filter to apply to the contents of the specified row(s). If unset, - // reads the entire table. - RowFilter filter = 5; - - // By default, rows are read sequentially, producing results which are - // guaranteed to arrive in increasing row order. Setting - // "allow_row_interleaving" to true allows multiple rows to be interleaved in - // the response stream, which increases throughput but breaks this guarantee, - // and may force the client to use more memory to buffer partially-received - // rows. Cannot be set to true when specifying "num_rows_limit". - bool allow_row_interleaving = 6; - - // The read will terminate after committing to N rows' worth of results. The - // default (zero) is to return all results. - // Note that "allow_row_interleaving" cannot be set to true when this is set. - int64 num_rows_limit = 7; -} - -// Response message for BigtableService.ReadRows. -message ReadRowsResponse { - // Specifies a piece of a row's contents returned as part of the read - // response stream. - message Chunk { - oneof chunk { - // A subset of the data from a particular row. As long as no "reset_row" - // is received in between, multiple "row_contents" from the same row are - // from the same atomic view of that row, and will be received in the - // expected family/column/timestamp order. - Family row_contents = 1; - - // Indicates that the client should drop all previous chunks for - // "row_key", as it will be re-read from the beginning. - bool reset_row = 2; - - // Indicates that the client can safely process all previous chunks for - // "row_key", as its data has been fully read. - bool commit_row = 3; - } - } - - // The key of the row for which we're receiving data. - // Results will be received in increasing row key order, unless - // "allow_row_interleaving" was specified in the request. - bytes row_key = 1; - - // One or more chunks of the row specified by "row_key". - repeated Chunk chunks = 2; -} - -// Request message for BigtableService.SampleRowKeys. -message SampleRowKeysRequest { - // The unique name of the table from which to sample row keys. - string table_name = 1; -} - -// Response message for BigtableService.SampleRowKeys. -message SampleRowKeysResponse { - // Sorted streamed sequence of sample row keys in the table. The table might - // have contents before the first row key in the list and after the last one, - // but a key containing the empty string indicates "end of table" and will be - // the last response given, if present. - // Note that row keys in this list may not have ever been written to or read - // from, and users should therefore not make any assumptions about the row key - // structure that are specific to their use case. - bytes row_key = 1; - - // Approximate total storage space used by all rows in the table which precede - // "row_key". Buffering the contents of all rows between two subsequent - // samples would require space roughly equal to the difference in their - // "offset_bytes" fields. - int64 offset_bytes = 2; -} - -// Request message for BigtableService.MutateRow. -message MutateRowRequest { - // The unique name of the table to which the mutation should be applied. - string table_name = 1; - - // The key of the row to which the mutation should be applied. - bytes row_key = 2; - - // Changes to be atomically applied to the specified row. Entries are applied - // in order, meaning that earlier mutations can be masked by later ones. - // Must contain at least one entry and at most 100000. - repeated Mutation mutations = 3; -} - -// Request message for BigtableService.MutateRows. -message MutateRowsRequest { - message Entry { - // The key of the row to which the `mutations` should be applied. - bytes row_key = 1; - - // Changes to be atomically applied to the specified row. Mutations are - // applied in order, meaning that earlier mutations can be masked by - // later ones. - // At least one mutation must be specified. - repeated Mutation mutations = 2; - } - - // The unique name of the table to which the mutations should be applied. - string table_name = 1; - - // The row keys/mutations to be applied in bulk. - // Each entry is applied as an atomic mutation, but the entries may be - // applied in arbitrary order (even between entries for the same row). - // At least one entry must be specified, and in total the entries may - // contain at most 100000 mutations. - repeated Entry entries = 2; -} - -// Response message for BigtableService.MutateRows. -message MutateRowsResponse { - // The results for each Entry from the request, presented in the order - // in which the entries were originally given. - repeated google.rpc.Status statuses = 1; -} - -// Request message for BigtableService.CheckAndMutateRowRequest -message CheckAndMutateRowRequest { - // The unique name of the table to which the conditional mutation should be - // applied. - string table_name = 1; - - // The key of the row to which the conditional mutation should be applied. - bytes row_key = 2; - - // The filter to be applied to the contents of the specified row. Depending - // on whether or not any results are yielded, either "true_mutations" or - // "false_mutations" will be executed. If unset, checks that the row contains - // any values at all. - RowFilter predicate_filter = 6; - - // Changes to be atomically applied to the specified row if "predicate_filter" - // yields at least one cell when applied to "row_key". Entries are applied in - // order, meaning that earlier mutations can be masked by later ones. - // Must contain at least one entry if "false_mutations" is empty, and at most - // 100000. - repeated Mutation true_mutations = 4; - - // Changes to be atomically applied to the specified row if "predicate_filter" - // does not yield any cells when applied to "row_key". Entries are applied in - // order, meaning that earlier mutations can be masked by later ones. - // Must contain at least one entry if "true_mutations" is empty, and at most - // 100000. - repeated Mutation false_mutations = 5; -} - -// Response message for BigtableService.CheckAndMutateRowRequest. -message CheckAndMutateRowResponse { - // Whether or not the request's "predicate_filter" yielded any results for - // the specified row. - bool predicate_matched = 1; -} - -// Request message for BigtableService.ReadModifyWriteRowRequest. -message ReadModifyWriteRowRequest { - // The unique name of the table to which the read/modify/write rules should be - // applied. - string table_name = 1; - - // The key of the row to which the read/modify/write rules should be applied. - bytes row_key = 2; - - // Rules specifying how the specified row's contents are to be transformed - // into writes. Entries are applied in order, meaning that earlier rules will - // affect the results of later ones. - repeated ReadModifyWriteRule rules = 3; -} diff --git a/gcloud_bigtable/_generated/_bigtable_table_data.proto b/gcloud_bigtable/_generated/_bigtable_table_data.proto deleted file mode 100644 index f81c878..0000000 --- a/gcloud_bigtable/_generated/_bigtable_table_data.proto +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.table.v1; - -import "google/longrunning/operations.proto"; -import "google/protobuf/duration.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableTableDataProto"; -option java_package = "com.google.bigtable.admin.table.v1"; - - -// A collection of user data indexed by row, column, and timestamp. -// Each table is served using the resources of its parent cluster. -message Table { - enum TimestampGranularity { - MILLIS = 0; - } - - // A unique identifier of the form - // /tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]* - string name = 1; - - // If this Table is in the process of being created, the Operation used to - // track its progress. As long as this operation is present, the Table will - // not accept any Table Admin or Read/Write requests. - google.longrunning.Operation current_operation = 2; - - // The column families configured for this table, mapped by column family id. - map column_families = 3; - - // The granularity (e.g. MILLIS, MICROS) at which timestamps are stored in - // this table. Timestamps not matching the granularity will be rejected. - // Cannot be changed once the table is created. - TimestampGranularity granularity = 4; -} - -// A set of columns within a table which share a common configuration. -message ColumnFamily { - // A unique identifier of the form /columnFamilies/[-_.a-zA-Z0-9]+ - // The last segment is the same as the "name" field in - // google.bigtable.v1.Family. - string name = 1; - - // Garbage collection expression specified by the following grammar: - // GC = EXPR - // | "" ; - // EXPR = EXPR, "||", EXPR (* lowest precedence *) - // | EXPR, "&&", EXPR - // | "(", EXPR, ")" (* highest precedence *) - // | PROP ; - // PROP = "version() >", NUM32 - // | "age() >", NUM64, [ UNIT ] ; - // NUM32 = non-zero-digit { digit } ; (* # NUM32 <= 2^32 - 1 *) - // NUM64 = non-zero-digit { digit } ; (* # NUM64 <= 2^63 - 1 *) - // UNIT = "d" | "h" | "m" (* d=days, h=hours, m=minutes, else micros *) - // GC expressions can be up to 500 characters in length - // - // The different types of PROP are defined as follows: - // version() - cell index, counting from most recent and starting at 1 - // age() - age of the cell (current time minus cell timestamp) - // - // Example: "version() > 3 || (age() > 3d && version() > 1)" - // drop cells beyond the most recent three, and drop cells older than three - // days unless they're the most recent cell in the row/column - // - // Garbage collection executes opportunistically in the background, and so - // it's possible for reads to return a cell even if it matches the active GC - // expression for its family. - string gc_expression = 2; - - // Garbage collection rule specified as a protobuf. - // Supersedes `gc_expression`. - // Must serialize to at most 500 bytes. - // - // NOTE: Garbage collection executes opportunistically in the background, and - // so it's possible for reads to return a cell even if it matches the active - // GC expression for its family. - GcRule gc_rule = 3; -} - -// Rule for determining which cells to delete during garbage collection. -message GcRule { - // A GcRule which deletes cells matching all of the given rules. - message Intersection { - // Only delete cells which would be deleted by every element of `rules`. - repeated GcRule rules = 1; - } - - // A GcRule which deletes cells matching any of the given rules. - message Union { - // Delete cells which would be deleted by any element of `rules`. - repeated GcRule rules = 1; - } - - oneof rule { - // Delete all cells in a column except the most recent N. - int32 max_num_versions = 1; - - // Delete cells in a column older than the given age. - // Values must be at least one millisecond, and will be truncated to - // microsecond granularity. - google.protobuf.Duration max_age = 2; - - // Delete cells that would be deleted by every nested rule. - Intersection intersection = 3; - - // Delete cells that would be deleted by any nested rule. - Union union = 4; - } -} diff --git a/gcloud_bigtable/_generated/_bigtable_table_service.proto b/gcloud_bigtable/_generated/_bigtable_table_service.proto deleted file mode 100644 index 417409c..0000000 --- a/gcloud_bigtable/_generated/_bigtable_table_service.proto +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.table.v1; - -import "google/api/annotations.proto"; -import "google/bigtable/admin/table/v1/bigtable_table_data.proto"; -import "google/bigtable/admin/table/v1/bigtable_table_service_messages.proto"; -import "google/protobuf/empty.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableTableServicesProto"; -option java_package = "com.google.bigtable.admin.table.v1"; - - -// Service for creating, configuring, and deleting Cloud Bigtable tables. -// Provides access to the table schemas only, not the data stored within the tables. -service BigtableTableService { - // Creates a new table, to be served from a specified cluster. - // The table can be created with a full set of initial column families, - // specified in the request. - rpc CreateTable(CreateTableRequest) returns (Table) { - option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*}/tables" body: "*" }; - } - - // Lists the names of all tables served from a specified cluster. - rpc ListTables(ListTablesRequest) returns (ListTablesResponse) { - option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*}/tables" }; - } - - // Gets the schema of the specified table, including its column families. - rpc GetTable(GetTableRequest) returns (Table) { - option (google.api.http) = { get: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}" }; - } - - // Permanently deletes a specified table and all of its data. - rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}" }; - } - - // Changes the name of a specified table. - // Cannot be used to move tables between clusters, zones, or projects. - rpc RenameTable(RenameTableRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename" body: "*" }; - } - - // Creates a new column family within a specified table. - rpc CreateColumnFamily(CreateColumnFamilyRequest) returns (ColumnFamily) { - option (google.api.http) = { post: "/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies" body: "*" }; - } - - // Changes the configuration of a specified column family. - rpc UpdateColumnFamily(ColumnFamily) returns (ColumnFamily) { - option (google.api.http) = { put: "/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}" body: "*" }; - } - - // Permanently deletes a specified column family and all of its data. - rpc DeleteColumnFamily(DeleteColumnFamilyRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { delete: "/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}" }; - } -} diff --git a/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto b/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto deleted file mode 100644 index 73f2a8c..0000000 --- a/gcloud_bigtable/_generated/_bigtable_table_service_messages.proto +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2015, Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.bigtable.admin.table.v1; - -import "google/bigtable/admin/table/v1/bigtable_table_data.proto"; - -option java_multiple_files = true; -option java_outer_classname = "BigtableTableServiceMessagesProto"; -option java_package = "com.google.bigtable.admin.table.v1"; - - -message CreateTableRequest { - // The unique name of the cluster in which to create the new table. - string name = 1; - - // The name by which the new table should be referred to within the cluster, - // e.g. "foobar" rather than "/tables/foobar". - string table_id = 2; - - // The Table to create. The `name` field of the Table and all of its - // ColumnFamilies must be left blank, and will be populated in the response. - Table table = 3; - - // The optional list of row keys that will be used to initially split the - // table into several tablets (Tablets are similar to HBase regions). - // Given two split keys, "s1" and "s2", three tablets will be created, - // spanning the key ranges: [, s1), [s1, s2), [s2, ). - // - // Example: - // * Row keys := ["a", "apple", "custom", "customer_1", "customer_2", - // "other", "zz"] - // * initial_split_keys := ["apple", "customer_1", "customer_2", "other"] - // * Key assignment: - // - Tablet 1 [, apple) => {"a"}. - // - Tablet 2 [apple, customer_1) => {"apple", "custom"}. - // - Tablet 3 [customer_1, customer_2) => {"customer_1"}. - // - Tablet 4 [customer_2, other) => {"customer_2"}. - // - Tablet 5 [other, ) => {"other", "zz"}. - repeated string initial_split_keys = 4; -} - -message ListTablesRequest { - // The unique name of the cluster for which tables should be listed. - string name = 1; -} - -message ListTablesResponse { - // The tables present in the requested cluster. - // At present, only the names of the tables are populated. - repeated Table tables = 1; -} - -message GetTableRequest { - // The unique name of the requested table. - string name = 1; -} - -message DeleteTableRequest { - // The unique name of the table to be deleted. - string name = 1; -} - -message RenameTableRequest { - // The current unique name of the table. - string name = 1; - - // The new name by which the table should be referred to within its containing - // cluster, e.g. "foobar" rather than "/tables/foobar". - string new_id = 2; -} - -message CreateColumnFamilyRequest { - // The unique name of the table in which to create the new column family. - string name = 1; - - // The name by which the new column family should be referred to within the - // table, e.g. "foobar" rather than "/columnFamilies/foobar". - string column_family_id = 2; - - // The column family to create. The `name` field must be left blank. - ColumnFamily column_family = 3; -} - -message DeleteColumnFamilyRequest { - // The unique name of the column family to be deleted. - string name = 1; -} diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py deleted file mode 100644 index 4106aab..0000000 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ /dev/null @@ -1,221 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto - -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto', - package='google.bigtable.admin.cluster.v1', - syntax='proto3', - serialized_pb=b'\n""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListClusters(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UndeleteCluster(self, request, context): - raise NotImplementedError() - -class BetaBigtableClusterServiceStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request, timeout): - raise NotImplementedError() - ListZones.future = None - @abc.abstractmethod - def GetCluster(self, request, timeout): - raise NotImplementedError() - GetCluster.future = None - @abc.abstractmethod - def ListClusters(self, request, timeout): - raise NotImplementedError() - ListClusters.future = None - @abc.abstractmethod - def CreateCluster(self, request, timeout): - raise NotImplementedError() - CreateCluster.future = None - @abc.abstractmethod - def UpdateCluster(self, request, timeout): - raise NotImplementedError() - UpdateCluster.future = None - @abc.abstractmethod - def DeleteCluster(self, request, timeout): - raise NotImplementedError() - DeleteCluster.future = None - @abc.abstractmethod - def UndeleteCluster(self, request, timeout): - raise NotImplementedError() - UndeleteCluster.future = None - -def beta_create_BigtableClusterService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.longrunning.operations_pb2 - request_deserializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - } - response_serializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - } - method_implementations = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): face_utilities.unary_unary_inline(servicer.CreateCluster), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): face_utilities.unary_unary_inline(servicer.DeleteCluster), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): face_utilities.unary_unary_inline(servicer.GetCluster), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): face_utilities.unary_unary_inline(servicer.ListClusters), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): face_utilities.unary_unary_inline(servicer.ListZones), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): face_utilities.unary_unary_inline(servicer.UndeleteCluster), - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): face_utilities.unary_unary_inline(servicer.UpdateCluster), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_BigtableClusterService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import google.longrunning.operations_pb2 - request_serializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - } - response_deserializers = { - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'CreateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'DeleteCluster'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'GetCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListClusters'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'ListZones'): gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UndeleteCluster'): google.longrunning.operations_pb2.Operation.FromString, - ('google.bigtable.admin.cluster.v1.BigtableClusterService', 'UpdateCluster'): gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - } - cardinalities = { - 'CreateCluster': cardinality.Cardinality.UNARY_UNARY, - 'DeleteCluster': cardinality.Cardinality.UNARY_UNARY, - 'GetCluster': cardinality.Cardinality.UNARY_UNARY, - 'ListClusters': cardinality.Cardinality.UNARY_UNARY, - 'ListZones': cardinality.Cardinality.UNARY_UNARY, - 'UndeleteCluster': cardinality.Cardinality.UNARY_UNARY, - 'UpdateCluster': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'google.bigtable.admin.cluster.v1.BigtableClusterService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_data_pb2.py b/gcloud_bigtable/_generated/bigtable_data_pb2.py deleted file mode 100644 index 47eb875..0000000 --- a/gcloud_bigtable/_generated/bigtable_data_pb2.py +++ /dev/null @@ -1,1226 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_data.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_data.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=b'\n&google/bigtable/v1/bigtable_data.proto\x12\x12google.bigtable.v1\"@\n\x03Row\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12,\n\x08\x66\x61milies\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Family\"C\n\x06\x46\x61mily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Column\"D\n\x06\x43olumn\x12\x11\n\tqualifier\x18\x01 \x01(\x0c\x12\'\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32\x18.google.bigtable.v1.Cell\"?\n\x04\x43\x65ll\x12\x18\n\x10timestamp_micros\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06labels\x18\x03 \x03(\t\".\n\x08RowRange\x12\x11\n\tstart_key\x18\x02 \x01(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\"L\n\x06RowSet\x12\x10\n\x08row_keys\x18\x01 \x03(\x0c\x12\x30\n\nrow_ranges\x18\x02 \x03(\x0b\x32\x1c.google.bigtable.v1.RowRange\"\xd6\x01\n\x0b\x43olumnRange\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12#\n\x19start_qualifier_inclusive\x18\x02 \x01(\x0cH\x00\x12#\n\x19start_qualifier_exclusive\x18\x03 \x01(\x0cH\x00\x12!\n\x17\x65nd_qualifier_inclusive\x18\x04 \x01(\x0cH\x01\x12!\n\x17\x65nd_qualifier_exclusive\x18\x05 \x01(\x0cH\x01\x42\x11\n\x0fstart_qualifierB\x0f\n\rend_qualifier\"N\n\x0eTimestampRange\x12\x1e\n\x16start_timestamp_micros\x18\x01 \x01(\x03\x12\x1c\n\x14\x65nd_timestamp_micros\x18\x02 \x01(\x03\"\xa8\x01\n\nValueRange\x12\x1f\n\x15start_value_inclusive\x18\x01 \x01(\x0cH\x00\x12\x1f\n\x15start_value_exclusive\x18\x02 \x01(\x0cH\x00\x12\x1d\n\x13\x65nd_value_inclusive\x18\x03 \x01(\x0cH\x01\x12\x1d\n\x13\x65nd_value_exclusive\x18\x04 \x01(\x0cH\x01\x42\r\n\x0bstart_valueB\x0b\n\tend_value\"\xdf\x08\n\tRowFilter\x12\x34\n\x05\x63hain\x18\x01 \x01(\x0b\x32#.google.bigtable.v1.RowFilter.ChainH\x00\x12>\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3' -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_ROW = _descriptor.Descriptor( - name='Row', - full_name='google.bigtable.v1.Row', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='google.bigtable.v1.Row.key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='families', full_name='google.bigtable.v1.Row.families', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=62, - serialized_end=126, -) - - -_FAMILY = _descriptor.Descriptor( - name='Family', - full_name='google.bigtable.v1.Family', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.v1.Family.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='columns', full_name='google.bigtable.v1.Family.columns', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=128, - serialized_end=195, -) - - -_COLUMN = _descriptor.Descriptor( - name='Column', - full_name='google.bigtable.v1.Column', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='qualifier', full_name='google.bigtable.v1.Column.qualifier', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells', full_name='google.bigtable.v1.Column.cells', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=197, - serialized_end=265, -) - - -_CELL = _descriptor.Descriptor( - name='Cell', - full_name='google.bigtable.v1.Cell', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='timestamp_micros', full_name='google.bigtable.v1.Cell.timestamp_micros', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.v1.Cell.value', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='labels', full_name='google.bigtable.v1.Cell.labels', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=267, - serialized_end=330, -) - - -_ROWRANGE = _descriptor.Descriptor( - name='RowRange', - full_name='google.bigtable.v1.RowRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_key', full_name='google.bigtable.v1.RowRange.start_key', index=0, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_key', full_name='google.bigtable.v1.RowRange.end_key', index=1, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=332, - serialized_end=378, -) - - -_ROWSET = _descriptor.Descriptor( - name='RowSet', - full_name='google.bigtable.v1.RowSet', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_keys', full_name='google.bigtable.v1.RowSet.row_keys', index=0, - number=1, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_ranges', full_name='google.bigtable.v1.RowSet.row_ranges', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=380, - serialized_end=456, -) - - -_COLUMNRANGE = _descriptor.Descriptor( - name='ColumnRange', - full_name='google.bigtable.v1.ColumnRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.ColumnRange.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_inclusive', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_exclusive', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_inclusive', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_exclusive', index=4, - number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='start_qualifier', full_name='google.bigtable.v1.ColumnRange.start_qualifier', - index=0, containing_type=None, fields=[]), - _descriptor.OneofDescriptor( - name='end_qualifier', full_name='google.bigtable.v1.ColumnRange.end_qualifier', - index=1, containing_type=None, fields=[]), - ], - serialized_start=459, - serialized_end=673, -) - - -_TIMESTAMPRANGE = _descriptor.Descriptor( - name='TimestampRange', - full_name='google.bigtable.v1.TimestampRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_timestamp_micros', full_name='google.bigtable.v1.TimestampRange.start_timestamp_micros', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_timestamp_micros', full_name='google.bigtable.v1.TimestampRange.end_timestamp_micros', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=675, - serialized_end=753, -) - - -_VALUERANGE = _descriptor.Descriptor( - name='ValueRange', - full_name='google.bigtable.v1.ValueRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_value_inclusive', full_name='google.bigtable.v1.ValueRange.start_value_inclusive', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_value_exclusive', full_name='google.bigtable.v1.ValueRange.start_value_exclusive', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_value_inclusive', full_name='google.bigtable.v1.ValueRange.end_value_inclusive', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_value_exclusive', full_name='google.bigtable.v1.ValueRange.end_value_exclusive', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='start_value', full_name='google.bigtable.v1.ValueRange.start_value', - index=0, containing_type=None, fields=[]), - _descriptor.OneofDescriptor( - name='end_value', full_name='google.bigtable.v1.ValueRange.end_value', - index=1, containing_type=None, fields=[]), - ], - serialized_start=756, - serialized_end=924, -) - - -_ROWFILTER_CHAIN = _descriptor.Descriptor( - name='Chain', - full_name='google.bigtable.v1.RowFilter.Chain', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='filters', full_name='google.bigtable.v1.RowFilter.Chain.filters', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1743, - serialized_end=1798, -) - -_ROWFILTER_INTERLEAVE = _descriptor.Descriptor( - name='Interleave', - full_name='google.bigtable.v1.RowFilter.Interleave', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='filters', full_name='google.bigtable.v1.RowFilter.Interleave.filters', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1800, - serialized_end=1860, -) - -_ROWFILTER_CONDITION = _descriptor.Descriptor( - name='Condition', - full_name='google.bigtable.v1.RowFilter.Condition', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='predicate_filter', full_name='google.bigtable.v1.RowFilter.Condition.predicate_filter', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='true_filter', full_name='google.bigtable.v1.RowFilter.Condition.true_filter', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='false_filter', full_name='google.bigtable.v1.RowFilter.Condition.false_filter', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1863, - serialized_end=2036, -) - -_ROWFILTER = _descriptor.Descriptor( - name='RowFilter', - full_name='google.bigtable.v1.RowFilter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='chain', full_name='google.bigtable.v1.RowFilter.chain', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='interleave', full_name='google.bigtable.v1.RowFilter.interleave', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='condition', full_name='google.bigtable.v1.RowFilter.condition', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sink', full_name='google.bigtable.v1.RowFilter.sink', index=3, - number=16, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pass_all_filter', full_name='google.bigtable.v1.RowFilter.pass_all_filter', index=4, - number=17, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='block_all_filter', full_name='google.bigtable.v1.RowFilter.block_all_filter', index=5, - number=18, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key_regex_filter', full_name='google.bigtable.v1.RowFilter.row_key_regex_filter', index=6, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_sample_filter', full_name='google.bigtable.v1.RowFilter.row_sample_filter', index=7, - number=14, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='family_name_regex_filter', full_name='google.bigtable.v1.RowFilter.family_name_regex_filter', index=8, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier_regex_filter', full_name='google.bigtable.v1.RowFilter.column_qualifier_regex_filter', index=9, - number=6, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_range_filter', full_name='google.bigtable.v1.RowFilter.column_range_filter', index=10, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='timestamp_range_filter', full_name='google.bigtable.v1.RowFilter.timestamp_range_filter', index=11, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value_regex_filter', full_name='google.bigtable.v1.RowFilter.value_regex_filter', index=12, - number=9, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value_range_filter', full_name='google.bigtable.v1.RowFilter.value_range_filter', index=13, - number=15, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_row_offset_filter', full_name='google.bigtable.v1.RowFilter.cells_per_row_offset_filter', index=14, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_row_limit_filter', full_name='google.bigtable.v1.RowFilter.cells_per_row_limit_filter', index=15, - number=11, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_column_limit_filter', full_name='google.bigtable.v1.RowFilter.cells_per_column_limit_filter', index=16, - number=12, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='strip_value_transformer', full_name='google.bigtable.v1.RowFilter.strip_value_transformer', index=17, - number=13, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='apply_label_transformer', full_name='google.bigtable.v1.RowFilter.apply_label_transformer', index=18, - number=19, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_ROWFILTER_CHAIN, _ROWFILTER_INTERLEAVE, _ROWFILTER_CONDITION, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='filter', full_name='google.bigtable.v1.RowFilter.filter', - index=0, containing_type=None, fields=[]), - ], - serialized_start=927, - serialized_end=2046, -) - - -_MUTATION_SETCELL = _descriptor.Descriptor( - name='SetCell', - full_name='google.bigtable.v1.Mutation.SetCell', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.SetCell.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.Mutation.SetCell.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='timestamp_micros', full_name='google.bigtable.v1.Mutation.SetCell.timestamp_micros', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.v1.Mutation.SetCell.value', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2344, - serialized_end=2441, -) - -_MUTATION_DELETEFROMCOLUMN = _descriptor.Descriptor( - name='DeleteFromColumn', - full_name='google.bigtable.v1.Mutation.DeleteFromColumn', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='time_range', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.time_range', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2443, - serialized_end=2564, -) - -_MUTATION_DELETEFROMFAMILY = _descriptor.Descriptor( - name='DeleteFromFamily', - full_name='google.bigtable.v1.Mutation.DeleteFromFamily', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromFamily.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2566, - serialized_end=2605, -) - -_MUTATION_DELETEFROMROW = _descriptor.Descriptor( - name='DeleteFromRow', - full_name='google.bigtable.v1.Mutation.DeleteFromRow', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2607, - serialized_end=2622, -) - -_MUTATION = _descriptor.Descriptor( - name='Mutation', - full_name='google.bigtable.v1.Mutation', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='set_cell', full_name='google.bigtable.v1.Mutation.set_cell', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_column', full_name='google.bigtable.v1.Mutation.delete_from_column', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_family', full_name='google.bigtable.v1.Mutation.delete_from_family', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_row', full_name='google.bigtable.v1.Mutation.delete_from_row', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_MUTATION_SETCELL, _MUTATION_DELETEFROMCOLUMN, _MUTATION_DELETEFROMFAMILY, _MUTATION_DELETEFROMROW, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='mutation', full_name='google.bigtable.v1.Mutation.mutation', - index=0, containing_type=None, fields=[]), - ], - serialized_start=2049, - serialized_end=2634, -) - - -_READMODIFYWRITERULE = _descriptor.Descriptor( - name='ReadModifyWriteRule', - full_name='google.bigtable.v1.ReadModifyWriteRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.ReadModifyWriteRule.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.ReadModifyWriteRule.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='append_value', full_name='google.bigtable.v1.ReadModifyWriteRule.append_value', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='increment_amount', full_name='google.bigtable.v1.ReadModifyWriteRule.increment_amount', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='rule', full_name='google.bigtable.v1.ReadModifyWriteRule.rule', - index=0, containing_type=None, fields=[]), - ], - serialized_start=2637, - serialized_end=2765, -) - -_ROW.fields_by_name['families'].message_type = _FAMILY -_FAMILY.fields_by_name['columns'].message_type = _COLUMN -_COLUMN.fields_by_name['cells'].message_type = _CELL -_ROWSET.fields_by_name['row_ranges'].message_type = _ROWRANGE -_COLUMNRANGE.oneofs_by_name['start_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['start_qualifier_inclusive']) -_COLUMNRANGE.fields_by_name['start_qualifier_inclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['start_qualifier'] -_COLUMNRANGE.oneofs_by_name['start_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['start_qualifier_exclusive']) -_COLUMNRANGE.fields_by_name['start_qualifier_exclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['start_qualifier'] -_COLUMNRANGE.oneofs_by_name['end_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['end_qualifier_inclusive']) -_COLUMNRANGE.fields_by_name['end_qualifier_inclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['end_qualifier'] -_COLUMNRANGE.oneofs_by_name['end_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['end_qualifier_exclusive']) -_COLUMNRANGE.fields_by_name['end_qualifier_exclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['end_qualifier'] -_VALUERANGE.oneofs_by_name['start_value'].fields.append( - _VALUERANGE.fields_by_name['start_value_inclusive']) -_VALUERANGE.fields_by_name['start_value_inclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['start_value'] -_VALUERANGE.oneofs_by_name['start_value'].fields.append( - _VALUERANGE.fields_by_name['start_value_exclusive']) -_VALUERANGE.fields_by_name['start_value_exclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['start_value'] -_VALUERANGE.oneofs_by_name['end_value'].fields.append( - _VALUERANGE.fields_by_name['end_value_inclusive']) -_VALUERANGE.fields_by_name['end_value_inclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['end_value'] -_VALUERANGE.oneofs_by_name['end_value'].fields.append( - _VALUERANGE.fields_by_name['end_value_exclusive']) -_VALUERANGE.fields_by_name['end_value_exclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['end_value'] -_ROWFILTER_CHAIN.fields_by_name['filters'].message_type = _ROWFILTER -_ROWFILTER_CHAIN.containing_type = _ROWFILTER -_ROWFILTER_INTERLEAVE.fields_by_name['filters'].message_type = _ROWFILTER -_ROWFILTER_INTERLEAVE.containing_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['predicate_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['true_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['false_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.containing_type = _ROWFILTER -_ROWFILTER.fields_by_name['chain'].message_type = _ROWFILTER_CHAIN -_ROWFILTER.fields_by_name['interleave'].message_type = _ROWFILTER_INTERLEAVE -_ROWFILTER.fields_by_name['condition'].message_type = _ROWFILTER_CONDITION -_ROWFILTER.fields_by_name['column_range_filter'].message_type = _COLUMNRANGE -_ROWFILTER.fields_by_name['timestamp_range_filter'].message_type = _TIMESTAMPRANGE -_ROWFILTER.fields_by_name['value_range_filter'].message_type = _VALUERANGE -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['chain']) -_ROWFILTER.fields_by_name['chain'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['interleave']) -_ROWFILTER.fields_by_name['interleave'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['condition']) -_ROWFILTER.fields_by_name['condition'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['sink']) -_ROWFILTER.fields_by_name['sink'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['pass_all_filter']) -_ROWFILTER.fields_by_name['pass_all_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['block_all_filter']) -_ROWFILTER.fields_by_name['block_all_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['row_key_regex_filter']) -_ROWFILTER.fields_by_name['row_key_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['row_sample_filter']) -_ROWFILTER.fields_by_name['row_sample_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['family_name_regex_filter']) -_ROWFILTER.fields_by_name['family_name_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['column_qualifier_regex_filter']) -_ROWFILTER.fields_by_name['column_qualifier_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['column_range_filter']) -_ROWFILTER.fields_by_name['column_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['timestamp_range_filter']) -_ROWFILTER.fields_by_name['timestamp_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['value_regex_filter']) -_ROWFILTER.fields_by_name['value_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['value_range_filter']) -_ROWFILTER.fields_by_name['value_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_row_offset_filter']) -_ROWFILTER.fields_by_name['cells_per_row_offset_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_row_limit_filter']) -_ROWFILTER.fields_by_name['cells_per_row_limit_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_column_limit_filter']) -_ROWFILTER.fields_by_name['cells_per_column_limit_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['strip_value_transformer']) -_ROWFILTER.fields_by_name['strip_value_transformer'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['apply_label_transformer']) -_ROWFILTER.fields_by_name['apply_label_transformer'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_MUTATION_SETCELL.containing_type = _MUTATION -_MUTATION_DELETEFROMCOLUMN.fields_by_name['time_range'].message_type = _TIMESTAMPRANGE -_MUTATION_DELETEFROMCOLUMN.containing_type = _MUTATION -_MUTATION_DELETEFROMFAMILY.containing_type = _MUTATION -_MUTATION_DELETEFROMROW.containing_type = _MUTATION -_MUTATION.fields_by_name['set_cell'].message_type = _MUTATION_SETCELL -_MUTATION.fields_by_name['delete_from_column'].message_type = _MUTATION_DELETEFROMCOLUMN -_MUTATION.fields_by_name['delete_from_family'].message_type = _MUTATION_DELETEFROMFAMILY -_MUTATION.fields_by_name['delete_from_row'].message_type = _MUTATION_DELETEFROMROW -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['set_cell']) -_MUTATION.fields_by_name['set_cell'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_column']) -_MUTATION.fields_by_name['delete_from_column'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_family']) -_MUTATION.fields_by_name['delete_from_family'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_row']) -_MUTATION.fields_by_name['delete_from_row'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_READMODIFYWRITERULE.oneofs_by_name['rule'].fields.append( - _READMODIFYWRITERULE.fields_by_name['append_value']) -_READMODIFYWRITERULE.fields_by_name['append_value'].containing_oneof = _READMODIFYWRITERULE.oneofs_by_name['rule'] -_READMODIFYWRITERULE.oneofs_by_name['rule'].fields.append( - _READMODIFYWRITERULE.fields_by_name['increment_amount']) -_READMODIFYWRITERULE.fields_by_name['increment_amount'].containing_oneof = _READMODIFYWRITERULE.oneofs_by_name['rule'] -DESCRIPTOR.message_types_by_name['Row'] = _ROW -DESCRIPTOR.message_types_by_name['Family'] = _FAMILY -DESCRIPTOR.message_types_by_name['Column'] = _COLUMN -DESCRIPTOR.message_types_by_name['Cell'] = _CELL -DESCRIPTOR.message_types_by_name['RowRange'] = _ROWRANGE -DESCRIPTOR.message_types_by_name['RowSet'] = _ROWSET -DESCRIPTOR.message_types_by_name['ColumnRange'] = _COLUMNRANGE -DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE -DESCRIPTOR.message_types_by_name['ValueRange'] = _VALUERANGE -DESCRIPTOR.message_types_by_name['RowFilter'] = _ROWFILTER -DESCRIPTOR.message_types_by_name['Mutation'] = _MUTATION -DESCRIPTOR.message_types_by_name['ReadModifyWriteRule'] = _READMODIFYWRITERULE - -Row = _reflection.GeneratedProtocolMessageType('Row', (_message.Message,), dict( - DESCRIPTOR = _ROW, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Row) - )) -_sym_db.RegisterMessage(Row) - -Family = _reflection.GeneratedProtocolMessageType('Family', (_message.Message,), dict( - DESCRIPTOR = _FAMILY, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Family) - )) -_sym_db.RegisterMessage(Family) - -Column = _reflection.GeneratedProtocolMessageType('Column', (_message.Message,), dict( - DESCRIPTOR = _COLUMN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Column) - )) -_sym_db.RegisterMessage(Column) - -Cell = _reflection.GeneratedProtocolMessageType('Cell', (_message.Message,), dict( - DESCRIPTOR = _CELL, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Cell) - )) -_sym_db.RegisterMessage(Cell) - -RowRange = _reflection.GeneratedProtocolMessageType('RowRange', (_message.Message,), dict( - DESCRIPTOR = _ROWRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowRange) - )) -_sym_db.RegisterMessage(RowRange) - -RowSet = _reflection.GeneratedProtocolMessageType('RowSet', (_message.Message,), dict( - DESCRIPTOR = _ROWSET, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowSet) - )) -_sym_db.RegisterMessage(RowSet) - -ColumnRange = _reflection.GeneratedProtocolMessageType('ColumnRange', (_message.Message,), dict( - DESCRIPTOR = _COLUMNRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ColumnRange) - )) -_sym_db.RegisterMessage(ColumnRange) - -TimestampRange = _reflection.GeneratedProtocolMessageType('TimestampRange', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMPRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.TimestampRange) - )) -_sym_db.RegisterMessage(TimestampRange) - -ValueRange = _reflection.GeneratedProtocolMessageType('ValueRange', (_message.Message,), dict( - DESCRIPTOR = _VALUERANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ValueRange) - )) -_sym_db.RegisterMessage(ValueRange) - -RowFilter = _reflection.GeneratedProtocolMessageType('RowFilter', (_message.Message,), dict( - - Chain = _reflection.GeneratedProtocolMessageType('Chain', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_CHAIN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Chain) - )) - , - - Interleave = _reflection.GeneratedProtocolMessageType('Interleave', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_INTERLEAVE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Interleave) - )) - , - - Condition = _reflection.GeneratedProtocolMessageType('Condition', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_CONDITION, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Condition) - )) - , - DESCRIPTOR = _ROWFILTER, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter) - )) -_sym_db.RegisterMessage(RowFilter) -_sym_db.RegisterMessage(RowFilter.Chain) -_sym_db.RegisterMessage(RowFilter.Interleave) -_sym_db.RegisterMessage(RowFilter.Condition) - -Mutation = _reflection.GeneratedProtocolMessageType('Mutation', (_message.Message,), dict( - - SetCell = _reflection.GeneratedProtocolMessageType('SetCell', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_SETCELL, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.SetCell) - )) - , - - DeleteFromColumn = _reflection.GeneratedProtocolMessageType('DeleteFromColumn', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMCOLUMN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromColumn) - )) - , - - DeleteFromFamily = _reflection.GeneratedProtocolMessageType('DeleteFromFamily', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMFAMILY, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromFamily) - )) - , - - DeleteFromRow = _reflection.GeneratedProtocolMessageType('DeleteFromRow', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMROW, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromRow) - )) - , - DESCRIPTOR = _MUTATION, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation) - )) -_sym_db.RegisterMessage(Mutation) -_sym_db.RegisterMessage(Mutation.SetCell) -_sym_db.RegisterMessage(Mutation.DeleteFromColumn) -_sym_db.RegisterMessage(Mutation.DeleteFromFamily) -_sym_db.RegisterMessage(Mutation.DeleteFromRow) - -ReadModifyWriteRule = _reflection.GeneratedProtocolMessageType('ReadModifyWriteRule', (_message.Message,), dict( - DESCRIPTOR = _READMODIFYWRITERULE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadModifyWriteRule) - )) -_sym_db.RegisterMessage(ReadModifyWriteRule) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\021BigtableDataProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py deleted file mode 100644 index 29d1d34..0000000 --- a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py +++ /dev/null @@ -1,678 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_service_messages.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 -from google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_service_messages.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=b'\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x17google/rpc/status.proto\"\x8b\x02\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x07row_set\x18\x08 \x01(\x0b\x32\x1a.google.bigtable.v1.RowSetH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xb0\x01\n\x11MutateRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12<\n\x07\x65ntries\x18\x02 \x03(\x0b\x32+.google.bigtable.v1.MutateRowsRequest.Entry\x1aI\n\x05\x45ntry\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12/\n\tmutations\x18\x02 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\":\n\x12MutateRowsResponse\x12$\n\x08statuses\x18\x01 \x03(\x0b\x32\x12.google.rpc.Status\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_rpc_dot_status__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_READROWSREQUEST = _descriptor.Descriptor( - name='ReadRowsRequest', - full_name='google.bigtable.v1.ReadRowsRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.ReadRowsRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadRowsRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_range', full_name='google.bigtable.v1.ReadRowsRequest.row_range', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_set', full_name='google.bigtable.v1.ReadRowsRequest.row_set', index=3, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filter', full_name='google.bigtable.v1.ReadRowsRequest.filter', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='allow_row_interleaving', full_name='google.bigtable.v1.ReadRowsRequest.allow_row_interleaving', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_rows_limit', full_name='google.bigtable.v1.ReadRowsRequest.num_rows_limit', index=6, - number=7, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='target', full_name='google.bigtable.v1.ReadRowsRequest.target', - index=0, containing_type=None, fields=[]), - ], - serialized_start=140, - serialized_end=407, -) - - -_READROWSRESPONSE_CHUNK = _descriptor.Descriptor( - name='Chunk', - full_name='google.bigtable.v1.ReadRowsResponse.Chunk', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_contents', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.row_contents', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='reset_row', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.reset_row', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='commit_row', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.commit_row', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='chunk', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.chunk', - index=0, containing_type=None, fields=[]), - ], - serialized_start=507, - serialized_end=618, -) - -_READROWSRESPONSE = _descriptor.Descriptor( - name='ReadRowsResponse', - full_name='google.bigtable.v1.ReadRowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadRowsResponse.row_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='chunks', full_name='google.bigtable.v1.ReadRowsResponse.chunks', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_READROWSRESPONSE_CHUNK, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=410, - serialized_end=618, -) - - -_SAMPLEROWKEYSREQUEST = _descriptor.Descriptor( - name='SampleRowKeysRequest', - full_name='google.bigtable.v1.SampleRowKeysRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.SampleRowKeysRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=620, - serialized_end=662, -) - - -_SAMPLEROWKEYSRESPONSE = _descriptor.Descriptor( - name='SampleRowKeysResponse', - full_name='google.bigtable.v1.SampleRowKeysResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.SampleRowKeysResponse.row_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='offset_bytes', full_name='google.bigtable.v1.SampleRowKeysResponse.offset_bytes', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=664, - serialized_end=726, -) - - -_MUTATEROWREQUEST = _descriptor.Descriptor( - name='MutateRowRequest', - full_name='google.bigtable.v1.MutateRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.MutateRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.MutateRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mutations', full_name='google.bigtable.v1.MutateRowRequest.mutations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=728, - serialized_end=832, -) - - -_MUTATEROWSREQUEST_ENTRY = _descriptor.Descriptor( - name='Entry', - full_name='google.bigtable.v1.MutateRowsRequest.Entry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.MutateRowsRequest.Entry.row_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mutations', full_name='google.bigtable.v1.MutateRowsRequest.Entry.mutations', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=938, - serialized_end=1011, -) - -_MUTATEROWSREQUEST = _descriptor.Descriptor( - name='MutateRowsRequest', - full_name='google.bigtable.v1.MutateRowsRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.MutateRowsRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='entries', full_name='google.bigtable.v1.MutateRowsRequest.entries', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_MUTATEROWSREQUEST_ENTRY, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=835, - serialized_end=1011, -) - - -_MUTATEROWSRESPONSE = _descriptor.Descriptor( - name='MutateRowsResponse', - full_name='google.bigtable.v1.MutateRowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='statuses', full_name='google.bigtable.v1.MutateRowsResponse.statuses', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1013, - serialized_end=1071, -) - - -_CHECKANDMUTATEROWREQUEST = _descriptor.Descriptor( - name='CheckAndMutateRowRequest', - full_name='google.bigtable.v1.CheckAndMutateRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.CheckAndMutateRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.CheckAndMutateRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='predicate_filter', full_name='google.bigtable.v1.CheckAndMutateRowRequest.predicate_filter', index=2, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='true_mutations', full_name='google.bigtable.v1.CheckAndMutateRowRequest.true_mutations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='false_mutations', full_name='google.bigtable.v1.CheckAndMutateRowRequest.false_mutations', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1074, - serialized_end=1303, -) - - -_CHECKANDMUTATEROWRESPONSE = _descriptor.Descriptor( - name='CheckAndMutateRowResponse', - full_name='google.bigtable.v1.CheckAndMutateRowResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='predicate_matched', full_name='google.bigtable.v1.CheckAndMutateRowResponse.predicate_matched', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1305, - serialized_end=1359, -) - - -_READMODIFYWRITEROWREQUEST = _descriptor.Descriptor( - name='ReadModifyWriteRowRequest', - full_name='google.bigtable.v1.ReadModifyWriteRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.rules', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1361, - serialized_end=1481, -) - -_READROWSREQUEST.fields_by_name['row_range'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWRANGE -_READROWSREQUEST.fields_by_name['row_set'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWSET -_READROWSREQUEST.fields_by_name['filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER -_READROWSREQUEST.oneofs_by_name['target'].fields.append( - _READROWSREQUEST.fields_by_name['row_key']) -_READROWSREQUEST.fields_by_name['row_key'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] -_READROWSREQUEST.oneofs_by_name['target'].fields.append( - _READROWSREQUEST.fields_by_name['row_range']) -_READROWSREQUEST.fields_by_name['row_range'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] -_READROWSREQUEST.oneofs_by_name['target'].fields.append( - _READROWSREQUEST.fields_by_name['row_set']) -_READROWSREQUEST.fields_by_name['row_set'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] -_READROWSRESPONSE_CHUNK.fields_by_name['row_contents'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._FAMILY -_READROWSRESPONSE_CHUNK.containing_type = _READROWSRESPONSE -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['row_contents']) -_READROWSRESPONSE_CHUNK.fields_by_name['row_contents'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['reset_row']) -_READROWSRESPONSE_CHUNK.fields_by_name['reset_row'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['commit_row']) -_READROWSRESPONSE_CHUNK.fields_by_name['commit_row'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE.fields_by_name['chunks'].message_type = _READROWSRESPONSE_CHUNK -_MUTATEROWREQUEST.fields_by_name['mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_MUTATEROWSREQUEST_ENTRY.fields_by_name['mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_MUTATEROWSREQUEST_ENTRY.containing_type = _MUTATEROWSREQUEST -_MUTATEROWSREQUEST.fields_by_name['entries'].message_type = _MUTATEROWSREQUEST_ENTRY -_MUTATEROWSRESPONSE.fields_by_name['statuses'].message_type = google_dot_rpc_dot_status__pb2._STATUS -_CHECKANDMUTATEROWREQUEST.fields_by_name['predicate_filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER -_CHECKANDMUTATEROWREQUEST.fields_by_name['true_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_CHECKANDMUTATEROWREQUEST.fields_by_name['false_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_READMODIFYWRITEROWREQUEST.fields_by_name['rules'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._READMODIFYWRITERULE -DESCRIPTOR.message_types_by_name['ReadRowsRequest'] = _READROWSREQUEST -DESCRIPTOR.message_types_by_name['ReadRowsResponse'] = _READROWSRESPONSE -DESCRIPTOR.message_types_by_name['SampleRowKeysRequest'] = _SAMPLEROWKEYSREQUEST -DESCRIPTOR.message_types_by_name['SampleRowKeysResponse'] = _SAMPLEROWKEYSRESPONSE -DESCRIPTOR.message_types_by_name['MutateRowRequest'] = _MUTATEROWREQUEST -DESCRIPTOR.message_types_by_name['MutateRowsRequest'] = _MUTATEROWSREQUEST -DESCRIPTOR.message_types_by_name['MutateRowsResponse'] = _MUTATEROWSRESPONSE -DESCRIPTOR.message_types_by_name['CheckAndMutateRowRequest'] = _CHECKANDMUTATEROWREQUEST -DESCRIPTOR.message_types_by_name['CheckAndMutateRowResponse'] = _CHECKANDMUTATEROWRESPONSE -DESCRIPTOR.message_types_by_name['ReadModifyWriteRowRequest'] = _READMODIFYWRITEROWREQUEST - -ReadRowsRequest = _reflection.GeneratedProtocolMessageType('ReadRowsRequest', (_message.Message,), dict( - DESCRIPTOR = _READROWSREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsRequest) - )) -_sym_db.RegisterMessage(ReadRowsRequest) - -ReadRowsResponse = _reflection.GeneratedProtocolMessageType('ReadRowsResponse', (_message.Message,), dict( - - Chunk = _reflection.GeneratedProtocolMessageType('Chunk', (_message.Message,), dict( - DESCRIPTOR = _READROWSRESPONSE_CHUNK, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsResponse.Chunk) - )) - , - DESCRIPTOR = _READROWSRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsResponse) - )) -_sym_db.RegisterMessage(ReadRowsResponse) -_sym_db.RegisterMessage(ReadRowsResponse.Chunk) - -SampleRowKeysRequest = _reflection.GeneratedProtocolMessageType('SampleRowKeysRequest', (_message.Message,), dict( - DESCRIPTOR = _SAMPLEROWKEYSREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.SampleRowKeysRequest) - )) -_sym_db.RegisterMessage(SampleRowKeysRequest) - -SampleRowKeysResponse = _reflection.GeneratedProtocolMessageType('SampleRowKeysResponse', (_message.Message,), dict( - DESCRIPTOR = _SAMPLEROWKEYSRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.SampleRowKeysResponse) - )) -_sym_db.RegisterMessage(SampleRowKeysResponse) - -MutateRowRequest = _reflection.GeneratedProtocolMessageType('MutateRowRequest', (_message.Message,), dict( - DESCRIPTOR = _MUTATEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowRequest) - )) -_sym_db.RegisterMessage(MutateRowRequest) - -MutateRowsRequest = _reflection.GeneratedProtocolMessageType('MutateRowsRequest', (_message.Message,), dict( - - Entry = _reflection.GeneratedProtocolMessageType('Entry', (_message.Message,), dict( - DESCRIPTOR = _MUTATEROWSREQUEST_ENTRY, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsRequest.Entry) - )) - , - DESCRIPTOR = _MUTATEROWSREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsRequest) - )) -_sym_db.RegisterMessage(MutateRowsRequest) -_sym_db.RegisterMessage(MutateRowsRequest.Entry) - -MutateRowsResponse = _reflection.GeneratedProtocolMessageType('MutateRowsResponse', (_message.Message,), dict( - DESCRIPTOR = _MUTATEROWSRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowsResponse) - )) -_sym_db.RegisterMessage(MutateRowsResponse) - -CheckAndMutateRowRequest = _reflection.GeneratedProtocolMessageType('CheckAndMutateRowRequest', (_message.Message,), dict( - DESCRIPTOR = _CHECKANDMUTATEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.CheckAndMutateRowRequest) - )) -_sym_db.RegisterMessage(CheckAndMutateRowRequest) - -CheckAndMutateRowResponse = _reflection.GeneratedProtocolMessageType('CheckAndMutateRowResponse', (_message.Message,), dict( - DESCRIPTOR = _CHECKANDMUTATEROWRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.CheckAndMutateRowResponse) - )) -_sym_db.RegisterMessage(CheckAndMutateRowResponse) - -ReadModifyWriteRowRequest = _reflection.GeneratedProtocolMessageType('ReadModifyWriteRowRequest', (_message.Message,), dict( - DESCRIPTOR = _READMODIFYWRITEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadModifyWriteRowRequest) - )) -_sym_db.RegisterMessage(ReadModifyWriteRowRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\034BigtableServiceMessagesProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py deleted file mode 100644 index e2fdb8a..0000000 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ /dev/null @@ -1,167 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_service.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 -from gcloud_bigtable._generated import bigtable_service_messages_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2 -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_service.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=b'\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xdd\x08\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xaa\x01\n\nMutateRows\x12%.google.bigtable.v1.MutateRowsRequest\x1a&.google.bigtable.v1.MutateRowsResponse\"M\x82\xd3\xe4\x93\x02G\"B/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}:mutateRows:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3' - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\026com.google.bigtable.v1B\025BigtableServicesProtoP\001\210\001\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaBigtableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def SampleRowKeys(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def MutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def MutateRows(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CheckAndMutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ReadModifyWriteRow(self, request, context): - raise NotImplementedError() - -class BetaBigtableServiceStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request, timeout): - raise NotImplementedError() - @abc.abstractmethod - def SampleRowKeys(self, request, timeout): - raise NotImplementedError() - @abc.abstractmethod - def MutateRow(self, request, timeout): - raise NotImplementedError() - MutateRow.future = None - @abc.abstractmethod - def MutateRows(self, request, timeout): - raise NotImplementedError() - MutateRows.future = None - @abc.abstractmethod - def CheckAndMutateRow(self, request, timeout): - raise NotImplementedError() - CheckAndMutateRow.future = None - @abc.abstractmethod - def ReadModifyWriteRow(self, request, timeout): - raise NotImplementedError() - ReadModifyWriteRow.future = None - -def beta_create_BigtableService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - request_deserializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.FromString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, - } - response_serializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, - } - method_implementations = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): face_utilities.unary_unary_inline(servicer.CheckAndMutateRow), - ('google.bigtable.v1.BigtableService', 'MutateRow'): face_utilities.unary_unary_inline(servicer.MutateRow), - ('google.bigtable.v1.BigtableService', 'MutateRows'): face_utilities.unary_unary_inline(servicer.MutateRows), - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): face_utilities.unary_unary_inline(servicer.ReadModifyWriteRow), - ('google.bigtable.v1.BigtableService', 'ReadRows'): face_utilities.unary_stream_inline(servicer.ReadRows), - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): face_utilities.unary_stream_inline(servicer.SampleRowKeys), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_BigtableService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - request_serializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, - } - response_deserializers = { - ('google.bigtable.v1.BigtableService', 'CheckAndMutateRow'): gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRow'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.v1.BigtableService', 'MutateRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowsResponse.FromString, - ('google.bigtable.v1.BigtableService', 'ReadModifyWriteRow'): gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, - ('google.bigtable.v1.BigtableService', 'ReadRows'): gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, - ('google.bigtable.v1.BigtableService', 'SampleRowKeys'): gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, - } - cardinalities = { - 'CheckAndMutateRow': cardinality.Cardinality.UNARY_UNARY, - 'MutateRow': cardinality.Cardinality.UNARY_UNARY, - 'MutateRows': cardinality.Cardinality.UNARY_UNARY, - 'ReadModifyWriteRow': cardinality.Cardinality.UNARY_UNARY, - 'ReadRows': cardinality.Cardinality.UNARY_STREAM, - 'SampleRowKeys': cardinality.Cardinality.UNARY_STREAM, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'google.bigtable.v1.BigtableService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py deleted file mode 100644 index fd47b56..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py +++ /dev/null @@ -1,377 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_data.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_data.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=b'\n8google/bigtable/admin/table/v1/bigtable_table_data.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a#google/longrunning/operations.proto\x1a\x1egoogle/protobuf/duration.proto\"\xfd\x02\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x11\x63urrent_operation\x18\x02 \x01(\x0b\x32\x1d.google.longrunning.Operation\x12R\n\x0f\x63olumn_families\x18\x03 \x03(\x0b\x32\x39.google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry\x12O\n\x0bgranularity\x18\x04 \x01(\x0e\x32:.google.bigtable.admin.table.v1.Table.TimestampGranularity\x1a\x63\n\x13\x43olumnFamiliesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily:\x02\x38\x01\"\"\n\x14TimestampGranularity\x12\n\n\x06MILLIS\x10\x00\"l\n\x0c\x43olumnFamily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rgc_expression\x18\x02 \x01(\t\x12\x37\n\x07gc_rule\x18\x03 \x01(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\"\xed\x02\n\x06GcRule\x12\x1a\n\x10max_num_versions\x18\x01 \x01(\x05H\x00\x12,\n\x07max_age\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12K\n\x0cintersection\x18\x03 \x01(\x0b\x32\x33.google.bigtable.admin.table.v1.GcRule.IntersectionH\x00\x12=\n\x05union\x18\x04 \x01(\x0b\x32,.google.bigtable.admin.table.v1.GcRule.UnionH\x00\x1a\x45\n\x0cIntersection\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\x1a>\n\x05Union\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRuleB\x06\n\x04ruleB>\n\"com.google.bigtable.admin.table.v1B\x16\x42igtableTableDataProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_longrunning_dot_operations__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - -_TABLE_TIMESTAMPGRANULARITY = _descriptor.EnumDescriptor( - name='TimestampGranularity', - full_name='google.bigtable.admin.table.v1.Table.TimestampGranularity', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='MILLIS', index=0, number=0, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=509, - serialized_end=543, -) -_sym_db.RegisterEnumDescriptor(_TABLE_TIMESTAMPGRANULARITY) - - -_TABLE_COLUMNFAMILIESENTRY = _descriptor.Descriptor( - name='ColumnFamiliesEntry', - full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), b'8\001'), - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=408, - serialized_end=507, -) - -_TABLE = _descriptor.Descriptor( - name='Table', - full_name='google.bigtable.admin.table.v1.Table', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.Table.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='current_operation', full_name='google.bigtable.admin.table.v1.Table.current_operation', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_families', full_name='google.bigtable.admin.table.v1.Table.column_families', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='granularity', full_name='google.bigtable.admin.table.v1.Table.granularity', index=3, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_TABLE_COLUMNFAMILIESENTRY, ], - enum_types=[ - _TABLE_TIMESTAMPGRANULARITY, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=162, - serialized_end=543, -) - - -_COLUMNFAMILY = _descriptor.Descriptor( - name='ColumnFamily', - full_name='google.bigtable.admin.table.v1.ColumnFamily', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.ColumnFamily.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='gc_expression', full_name='google.bigtable.admin.table.v1.ColumnFamily.gc_expression', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='gc_rule', full_name='google.bigtable.admin.table.v1.ColumnFamily.gc_rule', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=545, - serialized_end=653, -) - - -_GCRULE_INTERSECTION = _descriptor.Descriptor( - name='Intersection', - full_name='google.bigtable.admin.table.v1.GcRule.Intersection', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.admin.table.v1.GcRule.Intersection.rules', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=880, - serialized_end=949, -) - -_GCRULE_UNION = _descriptor.Descriptor( - name='Union', - full_name='google.bigtable.admin.table.v1.GcRule.Union', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.admin.table.v1.GcRule.Union.rules', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=951, - serialized_end=1013, -) - -_GCRULE = _descriptor.Descriptor( - name='GcRule', - full_name='google.bigtable.admin.table.v1.GcRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='max_num_versions', full_name='google.bigtable.admin.table.v1.GcRule.max_num_versions', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='max_age', full_name='google.bigtable.admin.table.v1.GcRule.max_age', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='intersection', full_name='google.bigtable.admin.table.v1.GcRule.intersection', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='union', full_name='google.bigtable.admin.table.v1.GcRule.union', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_GCRULE_INTERSECTION, _GCRULE_UNION, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='rule', full_name='google.bigtable.admin.table.v1.GcRule.rule', - index=0, containing_type=None, fields=[]), - ], - serialized_start=656, - serialized_end=1021, -) - -_TABLE_COLUMNFAMILIESENTRY.fields_by_name['value'].message_type = _COLUMNFAMILY -_TABLE_COLUMNFAMILIESENTRY.containing_type = _TABLE -_TABLE.fields_by_name['current_operation'].message_type = google_dot_longrunning_dot_operations__pb2._OPERATION -_TABLE.fields_by_name['column_families'].message_type = _TABLE_COLUMNFAMILIESENTRY -_TABLE.fields_by_name['granularity'].enum_type = _TABLE_TIMESTAMPGRANULARITY -_TABLE_TIMESTAMPGRANULARITY.containing_type = _TABLE -_COLUMNFAMILY.fields_by_name['gc_rule'].message_type = _GCRULE -_GCRULE_INTERSECTION.fields_by_name['rules'].message_type = _GCRULE -_GCRULE_INTERSECTION.containing_type = _GCRULE -_GCRULE_UNION.fields_by_name['rules'].message_type = _GCRULE -_GCRULE_UNION.containing_type = _GCRULE -_GCRULE.fields_by_name['max_age'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION -_GCRULE.fields_by_name['intersection'].message_type = _GCRULE_INTERSECTION -_GCRULE.fields_by_name['union'].message_type = _GCRULE_UNION -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['max_num_versions']) -_GCRULE.fields_by_name['max_num_versions'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['max_age']) -_GCRULE.fields_by_name['max_age'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['intersection']) -_GCRULE.fields_by_name['intersection'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['union']) -_GCRULE.fields_by_name['union'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -DESCRIPTOR.message_types_by_name['Table'] = _TABLE -DESCRIPTOR.message_types_by_name['ColumnFamily'] = _COLUMNFAMILY -DESCRIPTOR.message_types_by_name['GcRule'] = _GCRULE - -Table = _reflection.GeneratedProtocolMessageType('Table', (_message.Message,), dict( - - ColumnFamiliesEntry = _reflection.GeneratedProtocolMessageType('ColumnFamiliesEntry', (_message.Message,), dict( - DESCRIPTOR = _TABLE_COLUMNFAMILIESENTRY, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry) - )) - , - DESCRIPTOR = _TABLE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.Table) - )) -_sym_db.RegisterMessage(Table) -_sym_db.RegisterMessage(Table.ColumnFamiliesEntry) - -ColumnFamily = _reflection.GeneratedProtocolMessageType('ColumnFamily', (_message.Message,), dict( - DESCRIPTOR = _COLUMNFAMILY, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ColumnFamily) - )) -_sym_db.RegisterMessage(ColumnFamily) - -GcRule = _reflection.GeneratedProtocolMessageType('GcRule', (_message.Message,), dict( - - Intersection = _reflection.GeneratedProtocolMessageType('Intersection', (_message.Message,), dict( - DESCRIPTOR = _GCRULE_INTERSECTION, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule.Intersection) - )) - , - - Union = _reflection.GeneratedProtocolMessageType('Union', (_message.Message,), dict( - DESCRIPTOR = _GCRULE_UNION, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule.Union) - )) - , - DESCRIPTOR = _GCRULE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule) - )) -_sym_db.RegisterMessage(GcRule) -_sym_db.RegisterMessage(GcRule.Intersection) -_sym_db.RegisterMessage(GcRule.Union) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\026BigtableTableDataProtoP\001') -_TABLE_COLUMNFAMILIESENTRY.has_options = True -_TABLE_COLUMNFAMILIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), b'8\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py deleted file mode 100644 index f79cc23..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py +++ /dev/null @@ -1,389 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_service_messages.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import bigtable_table_data_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_service_messages.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=b'\nDgoogle/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\"\x86\x01\n\x12\x43reateTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08table_id\x18\x02 \x01(\t\x12\x34\n\x05table\x18\x03 \x01(\x0b\x32%.google.bigtable.admin.table.v1.Table\x12\x1a\n\x12initial_split_keys\x18\x04 \x03(\t\"!\n\x11ListTablesRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"K\n\x12ListTablesResponse\x12\x35\n\x06tables\x18\x01 \x03(\x0b\x32%.google.bigtable.admin.table.v1.Table\"\x1f\n\x0fGetTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\"\n\x12\x44\x65leteTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x12RenameTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\"\x88\x01\n\x19\x43reateColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_family_id\x18\x02 \x01(\t\x12\x43\n\rcolumn_family\x18\x03 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily\")\n\x19\x44\x65leteColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\tBI\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_CREATETABLEREQUEST = _descriptor.Descriptor( - name='CreateTableRequest', - full_name='google.bigtable.admin.table.v1.CreateTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.CreateTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='table_id', full_name='google.bigtable.admin.table.v1.CreateTableRequest.table_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='table', full_name='google.bigtable.admin.table.v1.CreateTableRequest.table', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='initial_split_keys', full_name='google.bigtable.admin.table.v1.CreateTableRequest.initial_split_keys', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=163, - serialized_end=297, -) - - -_LISTTABLESREQUEST = _descriptor.Descriptor( - name='ListTablesRequest', - full_name='google.bigtable.admin.table.v1.ListTablesRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.ListTablesRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=299, - serialized_end=332, -) - - -_LISTTABLESRESPONSE = _descriptor.Descriptor( - name='ListTablesResponse', - full_name='google.bigtable.admin.table.v1.ListTablesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='tables', full_name='google.bigtable.admin.table.v1.ListTablesResponse.tables', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=334, - serialized_end=409, -) - - -_GETTABLEREQUEST = _descriptor.Descriptor( - name='GetTableRequest', - full_name='google.bigtable.admin.table.v1.GetTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.GetTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=411, - serialized_end=442, -) - - -_DELETETABLEREQUEST = _descriptor.Descriptor( - name='DeleteTableRequest', - full_name='google.bigtable.admin.table.v1.DeleteTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.DeleteTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=444, - serialized_end=478, -) - - -_RENAMETABLEREQUEST = _descriptor.Descriptor( - name='RenameTableRequest', - full_name='google.bigtable.admin.table.v1.RenameTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.RenameTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_id', full_name='google.bigtable.admin.table.v1.RenameTableRequest.new_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=480, - serialized_end=530, -) - - -_CREATECOLUMNFAMILYREQUEST = _descriptor.Descriptor( - name='CreateColumnFamilyRequest', - full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_family_id', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.column_family_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_family', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.column_family', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=533, - serialized_end=669, -) - - -_DELETECOLUMNFAMILYREQUEST = _descriptor.Descriptor( - name='DeleteColumnFamilyRequest', - full_name='google.bigtable.admin.table.v1.DeleteColumnFamilyRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.DeleteColumnFamilyRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=671, - serialized_end=712, -) - -_CREATETABLEREQUEST.fields_by_name['table'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._TABLE -_LISTTABLESRESPONSE.fields_by_name['tables'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._TABLE -_CREATECOLUMNFAMILYREQUEST.fields_by_name['column_family'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._COLUMNFAMILY -DESCRIPTOR.message_types_by_name['CreateTableRequest'] = _CREATETABLEREQUEST -DESCRIPTOR.message_types_by_name['ListTablesRequest'] = _LISTTABLESREQUEST -DESCRIPTOR.message_types_by_name['ListTablesResponse'] = _LISTTABLESRESPONSE -DESCRIPTOR.message_types_by_name['GetTableRequest'] = _GETTABLEREQUEST -DESCRIPTOR.message_types_by_name['DeleteTableRequest'] = _DELETETABLEREQUEST -DESCRIPTOR.message_types_by_name['RenameTableRequest'] = _RENAMETABLEREQUEST -DESCRIPTOR.message_types_by_name['CreateColumnFamilyRequest'] = _CREATECOLUMNFAMILYREQUEST -DESCRIPTOR.message_types_by_name['DeleteColumnFamilyRequest'] = _DELETECOLUMNFAMILYREQUEST - -CreateTableRequest = _reflection.GeneratedProtocolMessageType('CreateTableRequest', (_message.Message,), dict( - DESCRIPTOR = _CREATETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.CreateTableRequest) - )) -_sym_db.RegisterMessage(CreateTableRequest) - -ListTablesRequest = _reflection.GeneratedProtocolMessageType('ListTablesRequest', (_message.Message,), dict( - DESCRIPTOR = _LISTTABLESREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ListTablesRequest) - )) -_sym_db.RegisterMessage(ListTablesRequest) - -ListTablesResponse = _reflection.GeneratedProtocolMessageType('ListTablesResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTTABLESRESPONSE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ListTablesResponse) - )) -_sym_db.RegisterMessage(ListTablesResponse) - -GetTableRequest = _reflection.GeneratedProtocolMessageType('GetTableRequest', (_message.Message,), dict( - DESCRIPTOR = _GETTABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GetTableRequest) - )) -_sym_db.RegisterMessage(GetTableRequest) - -DeleteTableRequest = _reflection.GeneratedProtocolMessageType('DeleteTableRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.DeleteTableRequest) - )) -_sym_db.RegisterMessage(DeleteTableRequest) - -RenameTableRequest = _reflection.GeneratedProtocolMessageType('RenameTableRequest', (_message.Message,), dict( - DESCRIPTOR = _RENAMETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.RenameTableRequest) - )) -_sym_db.RegisterMessage(RenameTableRequest) - -CreateColumnFamilyRequest = _reflection.GeneratedProtocolMessageType('CreateColumnFamilyRequest', (_message.Message,), dict( - DESCRIPTOR = _CREATECOLUMNFAMILYREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.CreateColumnFamilyRequest) - )) -_sym_db.RegisterMessage(CreateColumnFamilyRequest) - -DeleteColumnFamilyRequest = _reflection.GeneratedProtocolMessageType('DeleteColumnFamilyRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETECOLUMNFAMILYREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.DeleteColumnFamilyRequest) - )) -_sym_db.RegisterMessage(DeleteColumnFamilyRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\001') -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py deleted file mode 100644 index 4c27255..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ /dev/null @@ -1,203 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_service.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import bigtable_table_data_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2 -from gcloud_bigtable._generated import bigtable_table_service_messages_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2 -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_service.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=b'\n;google/bigtable/admin/table/v1/bigtable_table_service.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x1cgoogle/api/annotations.proto\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\x1a\x44google/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\x89\x0b\n\x14\x42igtableTableService\x12\xa4\x01\n\x0b\x43reateTable\x12\x32.google.bigtable.admin.table.v1.CreateTableRequest\x1a%.google.bigtable.admin.table.v1.Table\":\x82\xd3\xe4\x93\x02\x34\"//v1/{name=projects/*/zones/*/clusters/*}/tables:\x01*\x12\xac\x01\n\nListTables\x12\x31.google.bigtable.admin.table.v1.ListTablesRequest\x1a\x32.google.bigtable.admin.table.v1.ListTablesResponse\"7\x82\xd3\xe4\x93\x02\x31\x12//v1/{name=projects/*/zones/*/clusters/*}/tables\x12\x9d\x01\n\x08GetTable\x12/.google.bigtable.admin.table.v1.GetTableRequest\x1a%.google.bigtable.admin.table.v1.Table\"9\x82\xd3\xe4\x93\x02\x33\x12\x31/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x94\x01\n\x0b\x44\x65leteTable\x12\x32.google.bigtable.admin.table.v1.DeleteTableRequest\x1a\x16.google.protobuf.Empty\"9\x82\xd3\xe4\x93\x02\x33*1/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x9e\x01\n\x0bRenameTable\x12\x32.google.bigtable.admin.table.v1.RenameTableRequest\x1a\x16.google.protobuf.Empty\"C\x82\xd3\xe4\x93\x02=\"8/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename:\x01*\x12\xca\x01\n\x12\x43reateColumnFamily\x12\x39.google.bigtable.admin.table.v1.CreateColumnFamilyRequest\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"K\x82\xd3\xe4\x93\x02\x45\"@/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies:\x01*\x12\xbf\x01\n\x12UpdateColumnFamily\x12,.google.bigtable.admin.table.v1.ColumnFamily\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"M\x82\xd3\xe4\x93\x02G\x1a\x42/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}:\x01*\x12\xb3\x01\n\x12\x44\x65leteColumnFamily\x12\x39.google.bigtable.admin.table.v1.DeleteColumnFamilyRequest\x1a\x16.google.protobuf.Empty\"J\x82\xd3\xe4\x93\x02\x44*B/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}BB\n\"com.google.bigtable.admin.table.v1B\x1a\x42igtableTableServicesProtoP\x01\x62\x06proto3' - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), b'\n\"com.google.bigtable.admin.table.v1B\032BigtableTableServicesProtoP\001') -import abc -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaBigtableTableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListTables(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def RenameTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteColumnFamily(self, request, context): - raise NotImplementedError() - -class BetaBigtableTableServiceStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request, timeout): - raise NotImplementedError() - CreateTable.future = None - @abc.abstractmethod - def ListTables(self, request, timeout): - raise NotImplementedError() - ListTables.future = None - @abc.abstractmethod - def GetTable(self, request, timeout): - raise NotImplementedError() - GetTable.future = None - @abc.abstractmethod - def DeleteTable(self, request, timeout): - raise NotImplementedError() - DeleteTable.future = None - @abc.abstractmethod - def RenameTable(self, request, timeout): - raise NotImplementedError() - RenameTable.future = None - @abc.abstractmethod - def CreateColumnFamily(self, request, timeout): - raise NotImplementedError() - CreateColumnFamily.future = None - @abc.abstractmethod - def UpdateColumnFamily(self, request, timeout): - raise NotImplementedError() - UpdateColumnFamily.future = None - @abc.abstractmethod - def DeleteColumnFamily(self, request, timeout): - raise NotImplementedError() - DeleteColumnFamily.future = None - -def beta_create_BigtableTableService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - request_deserializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - } - response_serializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - } - method_implementations = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): face_utilities.unary_unary_inline(servicer.CreateColumnFamily), - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): face_utilities.unary_unary_inline(servicer.CreateTable), - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): face_utilities.unary_unary_inline(servicer.DeleteColumnFamily), - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): face_utilities.unary_unary_inline(servicer.DeleteTable), - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): face_utilities.unary_unary_inline(servicer.GetTable), - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): face_utilities.unary_unary_inline(servicer.ListTables), - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): face_utilities.unary_unary_inline(servicer.RenameTable), - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): face_utilities.unary_unary_inline(servicer.UpdateColumnFamily), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_BigtableTableService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import google.protobuf.empty_pb2 - request_serializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - } - response_deserializers = { - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'CreateTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteColumnFamily'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'DeleteTable'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'GetTable'): gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'ListTables'): gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'RenameTable'): google.protobuf.empty_pb2.Empty.FromString, - ('google.bigtable.admin.table.v1.BigtableTableService', 'UpdateColumnFamily'): gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - } - cardinalities = { - 'CreateColumnFamily': cardinality.Cardinality.UNARY_UNARY, - 'CreateTable': cardinality.Cardinality.UNARY_UNARY, - 'DeleteColumnFamily': cardinality.Cardinality.UNARY_UNARY, - 'DeleteTable': cardinality.Cardinality.UNARY_UNARY, - 'GetTable': cardinality.Cardinality.UNARY_UNARY, - 'ListTables': cardinality.Cardinality.UNARY_UNARY, - 'RenameTable': cardinality.Cardinality.UNARY_UNARY, - 'UpdateColumnFamily': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'google.bigtable.admin.table.v1.BigtableTableService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/operations_grpc_pb2.py b/gcloud_bigtable/_generated/operations_grpc_pb2.py deleted file mode 100644 index e4911b3..0000000 --- a/gcloud_bigtable/_generated/operations_grpc_pb2.py +++ /dev/null @@ -1,100 +0,0 @@ -import abc -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaOperationsServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListOperations(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CancelOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteOperation(self, request, context): - raise NotImplementedError() - -class BetaOperationsStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, timeout): - raise NotImplementedError() - GetOperation.future = None - @abc.abstractmethod - def ListOperations(self, request, timeout): - raise NotImplementedError() - ListOperations.future = None - @abc.abstractmethod - def CancelOperation(self, request, timeout): - raise NotImplementedError() - CancelOperation.future = None - @abc.abstractmethod - def DeleteOperation(self, request, timeout): - raise NotImplementedError() - DeleteOperation.future = None - -def beta_create_Operations_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - request_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.FromString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.FromString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.FromString, - } - response_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.SerializeToString, - } - method_implementations = { - ('google.longrunning.Operations', 'CancelOperation'): face_utilities.unary_unary_inline(servicer.CancelOperation), - ('google.longrunning.Operations', 'DeleteOperation'): face_utilities.unary_unary_inline(servicer.DeleteOperation), - ('google.longrunning.Operations', 'GetOperation'): face_utilities.unary_unary_inline(servicer.GetOperation), - ('google.longrunning.Operations', 'ListOperations'): face_utilities.unary_unary_inline(servicer.ListOperations), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_Operations_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - import google.longrunning.operations_pb2 - import google.protobuf.empty_pb2 - request_serializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.longrunning.operations_pb2.CancelOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'DeleteOperation'): google.longrunning.operations_pb2.DeleteOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.GetOperationRequest.SerializeToString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsRequest.SerializeToString, - } - response_deserializers = { - ('google.longrunning.Operations', 'CancelOperation'): google.protobuf.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'DeleteOperation'): google.protobuf.empty_pb2.Empty.FromString, - ('google.longrunning.Operations', 'GetOperation'): google.longrunning.operations_pb2.Operation.FromString, - ('google.longrunning.Operations', 'ListOperations'): google.longrunning.operations_pb2.ListOperationsResponse.FromString, - } - cardinalities = { - 'CancelOperation': cardinality.Cardinality.UNARY_UNARY, - 'DeleteOperation': cardinality.Cardinality.UNARY_UNARY, - 'GetOperation': cardinality.Cardinality.UNARY_UNARY, - 'ListOperations': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'google.longrunning.Operations', cardinalities, options=stub_options) diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py deleted file mode 100644 index e66251f..0000000 --- a/gcloud_bigtable/_grpc_mocks.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testing mocks for grpc.""" - - -class MethodMock(object): - """Mock for API method attached to a gRPC stub. - - In the beta implementation, these are of type. - :class:`grpc.framework.crust.implementations._UnaryUnaryMultiCallable` - """ - - def __init__(self, name, factory): - self._name = name - self._factory = factory - - def __call__(self, *args, **kwargs): - """Sync method meant to mock a gRPC stub request.""" - self._factory.method_calls.append((self._name, args, kwargs)) - curr_result, self._factory.results = (self._factory.results[0], - self._factory.results[1:]) - return curr_result - - -class StubMock(object): - """Class to act as a gPRC stub.""" - - def __init__(self, *results): - self.results = results - self.method_calls = [] - - def __getattr__(self, name): - # We need not worry about attributes set in constructor - # since __getattribute__ will handle them. - return MethodMock(name, self) diff --git a/gcloud_bigtable/_non_upstream_helpers.py b/gcloud_bigtable/_non_upstream_helpers.py deleted file mode 100644 index 4d053e7..0000000 --- a/gcloud_bigtable/_non_upstream_helpers.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helpers needed that are already present in gcloud-python. - -They are only needed here, but not needed for merging this upstream. -""" - - -import datetime -import os -import socket -import sys - -import pytz -import six - -from oauth2client.client import GoogleCredentials -from oauth2client.client import _get_application_default_credential_from_file -from oauth2client.service_account import ServiceAccountCredentials - -try: - from google.appengine.api import app_identity -except ImportError: - app_identity = None - - -PROJECT_ENV_VAR = 'GCLOUD_PROJECT' -"""Environment variable used to provide an implicit project ID.""" - -EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) - - -def _project_from_environment(): - """Attempts to get the project ID from an environment variable. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: The project ID provided or :data:`None` - """ - return os.getenv(PROJECT_ENV_VAR) - - -def _project_from_app_engine(): - """Gets the App Engine application ID if it can be inferred. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: App Engine application ID if running in App Engine, - else :data:`None`. - """ - if app_identity is None: - return None - - return app_identity.get_application_id() - - -def _project_from_compute_engine(): - """Gets the Compute Engine project ID if it can be inferred. - - Uses 169.254.169.254 for the metadata server to avoid request - latency from DNS lookup. - - See https://cloud.google.com/compute/docs/metadata#metadataserver - for information about this IP address. (This IP is also used for - Amazon EC2 instances, so the metadata flavor is crucial.) - - See https://github.com/google/oauth2client/issues/93 for context about - DNS latency. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: Compute Engine project ID if the metadata service is available, - else :data:`None`. - """ - host = '169.254.169.254' - uri_path = '/computeMetadata/v1/project/project-id' - headers = {'Metadata-Flavor': 'Google'} - connection = six.moves.http_client.HTTPConnection(host, timeout=0.1) - - try: - connection.request('GET', uri_path, headers=headers) - response = connection.getresponse() - if response.status == 200: - return response.read() - except socket.error: # socket.timeout or socket.error(64, 'Host is down') - pass - finally: - connection.close() - - -def _determine_project(project=None): - """Determine the project ID from the input or environment. - - When checking the environment, the following precedence is observed: - - * GCLOUD_PROJECT environment variable - * Google App Engine application ID - * Google Compute Engine project ID (from metadata server) - - :type project: str - :param project: (Optional) The ID of the project which owns the - clusters, tables and data. If not provided, will attempt - to determine from the environment. - - :rtype: str - :returns: The project ID provided or inferred from the environment. - :raises: :class:`EnvironmentError` if the project ID was not - passed in and can't be inferred from the environment. - """ - if project is None: - project = _project_from_environment() - - if project is None: - project = _project_from_app_engine() - - if project is None: - project = _project_from_compute_engine() - - if project is None: - raise EnvironmentError('Project ID was not provided and could not ' - 'be determined from environment.') - - return project - - -def _get_contents(filename): - """Get the contents of a file. - - This is just implemented so we can stub out while testing. - - :type filename: :class:`str` or :func:`unicode ` - :param filename: The name of a file to open. - - :rtype: bytes - :returns: The bytes loaded from the file. - """ - with open(filename, 'rb') as file_obj: - return file_obj.read() - - -def get_credentials(): - """Get default credentials.""" - return GoogleCredentials.get_application_default() - - -class _FactoryMixin(object): - """Mixin for factories to get credentials.""" - - def __init__(self, project=None, credentials=None): - self.project = _determine_project(project) - self._credentials = credentials - - @classmethod - def from_service_account_json(cls, json_credentials_path, *args, **kwargs): - """Factory to retrieve JSON credentials while creating client object. - - :type json_credentials_path: str - :param json_credentials_path: The path to a private key file (this file - was given to you when you created the - service account). This file must contain - a JSON object with a private key and - other credentials information (downloaded - from the Google APIs console). - - :type args: tuple - :param args: Positional arguments. - - :type kwargs: dict - :param kwargs: Keyword arguments. - - :rtype: :class:`_FactoryMixin` - :returns: The client created with the retrieved JSON credentials. - """ - credentials = _get_application_default_credential_from_file( - json_credentials_path) - kwargs['credentials'] = credentials - return cls(*args, **kwargs) - - @classmethod - def from_service_account_p12(cls, client_email, private_key_path, - *args, **kwargs): - """Factory to retrieve P12 credentials while creating client object. - - .. note:: - - Unless you have an explicit reason to use a PKCS12 key for your - service account, we recommend using a JSON key. - - :type client_email: str - :param client_email: The e-mail attached to the service account. - - :type private_key_path: str - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). This file must be in P12 format. - - :type args: tuple - :param args: Positional arguments. - - :type kwargs: dict - :param kwargs: Keyword arguments. - - :rtype: :class:`_FactoryMixin` - :returns: The client created with the retrieved P12 credentials. - """ - credentials = ServiceAccountCredentials.from_p12_keyfile( - client_email, private_key_path) - kwargs['credentials'] = credentials - return cls(*args, **kwargs) - - -def _timestamp_to_microseconds(timestamp, granularity=1000): - """Converts a native datetime object to microseconds. - - .. note:: - - If ``timestamp`` does not have the same timezone as ``EPOCH`` - (which is UTC), then subtracting the epoch from the timestamp - will raise a :class:`TypeError `. - - :type timestamp: :class:`datetime.datetime` - :param timestamp: A timestamp to be converted to microseconds. - - :type granularity: int - :param granularity: The resolution (relative to microseconds) that the - timestamp should be truncated to. Defaults to 1000 - and no other value is likely needed since the only - value of the enum - :class:`.data_pb2.Table.TimestampGranularity` is - :data:`.data_pb2.Table.MILLIS`. - - :rtype: int - :returns: The ``timestamp`` as microseconds (with the appropriate - granularity). - """ - timestamp_seconds = (timestamp - EPOCH).total_seconds() - timestamp_micros = int(10**6 * timestamp_seconds) - # Truncate to granularity. - timestamp_micros -= (timestamp_micros % granularity) - return timestamp_micros - - -def _microseconds_to_timestamp(microseconds): - """Converts microseconds to a native datetime object. - - :type microseconds: int - :param microseconds: The ``timestamp`` as microseconds. - - :rtype: :class:`datetime.datetime` - :returns: A timestamp to be converted from microseconds. - """ - return EPOCH + datetime.timedelta(microseconds=microseconds) - - -def _to_bytes(value, encoding='ascii'): - """Converts a string value to bytes, if necessary. - - Unfortunately, ``six.b`` is insufficient for this task since in - Python2 it does not modify ``unicode`` objects. - - :type value: str / bytes or unicode - :param value: The string/bytes value to be converted. - - :type encoding: str - :param encoding: The encoding to use to convert unicode to bytes. Defaults - to "ascii", which will not allow any characters from - ordinals larger than 127. Other useful values are - "latin-1", which which will only allows byte ordinals - (up to 255) and "utf-8", which will encode any unicode - that needs to be. - - :rtype: str / bytes - :returns: The original value converted to bytes (if unicode) or as passed - in if it started out as bytes. - :raises: :class:`TypeError ` if the value - could not be converted to bytes. - """ - result = (value.encode(encoding) - if isinstance(value, six.text_type) else value) - if isinstance(result, six.binary_type): - return result - else: - raise TypeError('%r could not be converted to bytes' % (value,)) - - -def _total_seconds_backport(offset): - """Backport of timedelta.total_seconds() from python 2.7+. - - :type offset: :class:`datetime.timedelta` - :param offset: A timedelta object. - - :rtype: int - :returns: The total seconds (including microseconds) in the - duration. - """ - seconds = offset.days * 24 * 60 * 60 + offset.seconds - return seconds + offset.microseconds * 1e-6 - - -def _total_seconds(offset): - """Version independent total seconds for a time delta. - - :type offset: :class:`datetime.timedelta` - :param offset: A timedelta object. - - :rtype: int - :returns: The total seconds (including microseconds) in the - duration. - """ - if sys.version_info[:2] < (2, 7): # pragma: NO COVER - return _total_seconds_backport(offset) - else: - return offset.total_seconds() - - -def _pb_timestamp_to_datetime(timestamp): - """Convert a Timestamp protobuf to a datetime object. - - :type timestamp: :class:`google.protobuf.timestamp_pb2.Timestamp` - :param timestamp: A Google returned timestamp protobuf. - - :rtype: :class:`datetime.datetime` - :returns: A UTC datetime object converted from a protobuf timestamp. - """ - return ( - EPOCH + - datetime.timedelta( - seconds=timestamp.seconds, - microseconds=(timestamp.nanos / 1000.0), - ) - ) diff --git a/gcloud_bigtable/_testing.py b/gcloud_bigtable/_testing.py deleted file mode 100644 index 46d9609..0000000 --- a/gcloud_bigtable/_testing.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared testing utilities.""" - - -class _MockCalled(object): - - def __init__(self, result=None): - self.called_args = [] - self.called_kwargs = [] - self.result = result - - def check_called(self, test_case, args_list, kwargs_list=None): - test_case.assertEqual(self.called_args, args_list) - if kwargs_list is None: - test_case.assertTrue(all([val == {} - for val in self.called_kwargs])) - else: - test_case.assertEqual(self.called_kwargs, kwargs_list) - - def __call__(self, *args, **kwargs): - self.called_args.append(args) - self.called_kwargs.append(kwargs) - return self.result - - -class _AttachedMethod(object): - - def __init__(self, parent, name): - self.parent = parent - self.name = name - - def __call__(self, *args, **kwargs): - self.parent._called.append((self.name, args, kwargs)) - curr_result = self.parent._results[0] - self.parent._results = self.parent._results[1:] - return curr_result - - -class _MockWithAttachedMethods(object): - - def __init__(self, *results): - self._results = results - self._called = [] - - def __getattr__(self, name): - # We need not worry about names: _results, _called - # since __getattribute__ will handle them. - return _AttachedMethod(self, name) - - -class _Monkey(object): - # context-manager for replacing module names in the scope of a test. - - def __init__(self, module, **kw): - self.module = module - self.to_restore = dict([(key, getattr(module, key)) for key in kw]) - for key, value in kw.items(): - setattr(module, key, value) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for key, value in self.to_restore.items(): - setattr(self.module, key, value) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py deleted file mode 100644 index 7e12455..0000000 --- a/gcloud_bigtable/client.py +++ /dev/null @@ -1,475 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Parent client for calling the Google Cloud Bigtable API. - -This is the base from which all interactions with the API occur. - -In the hierarchy of API concepts - -* a :class:`Client` owns a :class:`.Cluster` -* a :class:`.Cluster` owns a :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily <.column_family.ColumnFamily>` -* a :class:`Table ` owns a :class:`Row <.row.Row>` - (and all the cells in the row) -""" - - -import copy - -from grpc.beta import implementations - -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import bigtable_cluster_service_pb2 -from gcloud_bigtable._generated import bigtable_service_pb2 -from gcloud_bigtable._generated import bigtable_table_service_pb2 -from gcloud_bigtable._generated import operations_grpc_pb2 -from gcloud_bigtable._non_upstream_helpers import _FactoryMixin -from gcloud_bigtable._non_upstream_helpers import get_credentials -from gcloud_bigtable.cluster import Cluster - - -TABLE_STUB_FACTORY = ( - bigtable_table_service_pb2.beta_create_BigtableTableService_stub) -TABLE_ADMIN_HOST = 'bigtabletableadmin.googleapis.com' -"""Table Admin API request host.""" -TABLE_ADMIN_PORT = 443 -"""Table Admin API request port.""" - -CLUSTER_STUB_FACTORY = ( - bigtable_cluster_service_pb2.beta_create_BigtableClusterService_stub) -CLUSTER_ADMIN_HOST = 'bigtableclusteradmin.googleapis.com' -"""Cluster Admin API request host.""" -CLUSTER_ADMIN_PORT = 443 -"""Cluster Admin API request port.""" - -DATA_STUB_FACTORY = bigtable_service_pb2.beta_create_BigtableService_stub -DATA_API_HOST = 'bigtable.googleapis.com' -"""Data API request host.""" -DATA_API_PORT = 443 -"""Data API request port.""" - -OPERATIONS_STUB_FACTORY = operations_grpc_pb2.beta_create_Operations_stub - -ADMIN_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.admin' -"""Scope for interacting with the Cluster Admin and Table Admin APIs.""" -DATA_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.data' -"""Scope for reading and writing table data.""" -READ_ONLY_SCOPE = ('https://www.googleapis.com/auth/' - 'cloud-bigtable.data.readonly') -"""Scope for reading table data.""" - -DEFAULT_TIMEOUT_SECONDS = 10 -"""The default timeout to use for API requests.""" - -DEFAULT_USER_AGENT = 'gcloud-bigtable-python' -"""The default user agent for API requests.""" - - -class Client(_FactoryMixin): - """Client for interacting with Google Cloud Bigtable API. - - .. note:: - - Since the Cloud Bigtable API requires the gRPC transport, no - ``http`` argument is accepted by this class. - - :type project: :class:`str` or :func:`unicode ` - :param project: (Optional) The ID of the project which owns the - clusters, tables and data. If not provided, will - attempt to determine from the environment. - - :type credentials: - :class:`OAuth2Credentials ` or - :data:`NoneType ` - :param credentials: (Optional) The OAuth2 Credentials to use for this - cluster. If not provided, defaults to the Google - Application Default Credentials. - - :type read_only: bool - :param read_only: (Optional) Boolean indicating if the data scope should be - for reading only (or for writing as well). Defaults to - :data:`False`. - - :type admin: bool - :param admin: (Optional) Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. This - requires the :const:`ADMIN_SCOPE`. Defaults to :data:`False`. - - :type user_agent: str - :param user_agent: (Optional) The user agent to be used with API request. - Defaults to :const:`DEFAULT_USER_AGENT`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. If not - passed, defaults to - :const:`DEFAULT_TIMEOUT_SECONDS`. - - :raises: :class:`ValueError ` if both ``read_only`` - and ``admin`` are :data:`True` - """ - - def __init__(self, project=None, credentials=None, - read_only=False, admin=False, user_agent=DEFAULT_USER_AGENT, - timeout_seconds=DEFAULT_TIMEOUT_SECONDS): - super(Client, self).__init__(project=project) - if credentials is None: - credentials = get_credentials() - - if read_only and admin: - raise ValueError('A read-only client cannot also perform' - 'administrative actions.') - - scopes = [] - if read_only: - scopes.append(READ_ONLY_SCOPE) - else: - scopes.append(DATA_SCOPE) - - if admin: - scopes.append(ADMIN_SCOPE) - - self._admin = bool(admin) - self._credentials = credentials.create_scoped(scopes) - self.user_agent = user_agent - self.timeout_seconds = timeout_seconds - - # These will be set in start(). - self._data_stub_internal = None - self._cluster_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None - - def copy(self): - """Make a copy of this client. - - Copies the local data stored as simple types but does not copy the - current state of any open connections with the Cloud Bigtable API. - - :rtype: :class:`.Client` - :returns: A copy of the current client. - """ - copied_creds = copy.deepcopy(self._credentials) - return self.__class__( - self.project, - copied_creds, - READ_ONLY_SCOPE in copied_creds.scopes, - self._admin, - self.user_agent, - self.timeout_seconds, - ) - - @property - def credentials(self): - """Getter for client's credentials. - - :rtype: - :class:`OAuth2Credentials ` - :returns: The credentials stored on the client. - """ - return self._credentials - - @property - def project_name(self): - """Project name to be used with Cluster Admin API. - - .. note:: - - This property will not change if ``project`` does not, but the - return value is not cached. - - The project name is of the form - - ``"projects/{project}"`` - - :rtype: str - :returns: The project name to be used with the Cloud Bigtable Admin - API RPC service. - """ - return 'projects/' + self.project - - @property - def _data_stub(self): - """Getter for the gRPC stub used for the Data API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client has not been :meth:`start`-ed. - """ - if self._data_stub_internal is None: - raise ValueError('Client has not been started.') - return self._data_stub_internal - - @property - def _cluster_stub(self): - """Getter for the gRPC stub used for the Cluster Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._cluster_stub_internal is None: - raise ValueError('Client has not been started.') - return self._cluster_stub_internal - - @property - def _operations_stub(self): - """Getter for the gRPC stub used for the Operations API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._operations_stub_internal is None: - raise ValueError('Client has not been started.') - return self._operations_stub_internal - - @property - def _table_stub(self): - """Getter for the gRPC stub used for the Table Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._table_stub_internal is None: - raise ValueError('Client has not been started.') - return self._table_stub_internal - - def _make_data_stub(self): - """Creates gRPC stub to make requests to the Data API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return _make_stub(self, DATA_STUB_FACTORY, - DATA_API_HOST, DATA_API_PORT) - - def _make_cluster_stub(self): - """Creates gRPC stub to make requests to the Cluster Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return _make_stub(self, CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) - - def _make_operations_stub(self): - """Creates gRPC stub to make requests to the Operations API. - - These are for long-running operations of the Cluster Admin API, - hence the host and port matching. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return _make_stub(self, OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) - - def _make_table_stub(self): - """Creates gRPC stub to make requests to the Table Admin API. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: A gRPC stub object. - """ - return _make_stub(self, TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) - - def is_started(self): - """Check if the client has been started. - - :rtype: bool - :returns: Boolean indicating if the client has been started. - """ - return self._data_stub_internal is not None - - def start(self): - """Prepare the client to make requests. - - Activates gRPC contexts for making requests to the Bigtable - Service(s). - """ - if self.is_started(): - return - - # NOTE: We __enter__ the stubs more-or-less permanently. This is - # because only after entering the context managers is the - # connection created. We don't want to immediately close - # those connections since the client will make many - # requests with it over HTTP/2. - self._data_stub_internal = self._make_data_stub() - self._data_stub_internal.__enter__() - if self._admin: - self._cluster_stub_internal = self._make_cluster_stub() - self._operations_stub_internal = self._make_operations_stub() - self._table_stub_internal = self._make_table_stub() - - self._cluster_stub_internal.__enter__() - self._operations_stub_internal.__enter__() - self._table_stub_internal.__enter__() - - def stop(self): - """Closes all the open gRPC clients.""" - if not self.is_started(): - return - - # When exit-ing, we pass None as the exception type, value and - # traceback to __exit__. - self._data_stub_internal.__exit__(None, None, None) - if self._admin: - self._cluster_stub_internal.__exit__(None, None, None) - self._operations_stub_internal.__exit__(None, None, None) - self._table_stub_internal.__exit__(None, None, None) - - self._data_stub_internal = None - self._cluster_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None - - def cluster(self, zone, cluster_id, display_name=None, serve_nodes=3): - """Factory to create a cluster associated with this client. - - :type zone: str - :param zone: The name of the zone where the cluster resides. - - :type cluster_id: str - :param cluster_id: The ID of the cluster. - - :type display_name: str - :param display_name: (Optional) The display name for the cluster in the - Cloud Console UI. (Must be between 4 and 30 - characters.) If this value is not set in the - constructor, will fall back to the cluster ID. - - :type serve_nodes: int - :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3. - - :rtype: :class:`.Cluster` - :returns: The cluster owned by this client. - """ - return Cluster(zone, cluster_id, self, - display_name=display_name, serve_nodes=serve_nodes) - - def list_zones(self): - """Lists zones associated with project. - - :rtype: list - :returns: The names (as :class:`str`) of the zones - :raises: :class:`ValueError ` if one of the - zones is not in ``OK`` state. - """ - request_pb = messages_pb2.ListZonesRequest(name=self.project_name) - # We expect a `.messages_pb2.ListZonesResponse` - list_zones_response = self._cluster_stub.ListZones( - request_pb, self.timeout_seconds) - - result = [] - for zone in list_zones_response.zones: - if zone.status != data_pb2.Zone.OK: - raise ValueError('Zone %s not in OK state' % ( - zone.display_name,)) - result.append(zone.display_name) - return result - - def list_clusters(self): - """Lists clusters owned by the project. - - :rtype: tuple - :returns: A pair of results, the first is a list of :class:`.Cluster` s - returned and the second is a list of strings (the failed - zones in the request). - """ - request_pb = messages_pb2.ListClustersRequest(name=self.project_name) - # We expect a `.messages_pb2.ListClustersResponse` - list_clusters_response = self._cluster_stub.ListClusters( - request_pb, self.timeout_seconds) - - failed_zones = [zone.display_name - for zone in list_clusters_response.failed_zones] - clusters = [Cluster.from_pb(cluster_pb, self) - for cluster_pb in list_clusters_response.clusters] - return clusters, failed_zones - - -class _MetadataPlugin(object): - """Callable class to transform metadata for gRPC requests. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - """ - - def __init__(self, client): - self._credentials = client.credentials - self._user_agent = client.user_agent - - def __call__(self, unused_context, callback): - """Adds authorization header to request metadata.""" - access_token = self._credentials.get_access_token().access_token - headers = [ - ('Authorization', 'Bearer ' + access_token), - ('User-agent', self._user_agent), - ] - callback(headers, None) - - -def _make_stub(client, stub_factory, host, port): - """Makes a stub for an RPC service. - - Uses / depends on the beta implementation of gRPC. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - - :type stub_factory: callable - :param stub_factory: A factory which will create a gRPC stub for - a given service. - - :type host: str - :param host: The host for the service. - - :type port: int - :param port: The port for the service. - - :rtype: :class:`grpc.beta._stub._AutoIntermediary` - :returns: The stub object used to make gRPC requests to a given API. - """ - # Leaving the first argument to ssl_channel_credentials() as None - # loads root certificates from `grpc/_adapter/credentials/roots.pem`. - transport_creds = implementations.ssl_channel_credentials(None, None, None) - custom_metadata_plugin = _MetadataPlugin(client) - auth_creds = implementations.metadata_call_credentials( - custom_metadata_plugin, name='google_creds') - channel_creds = implementations.composite_channel_credentials( - transport_creds, auth_creds) - channel = implementations.secure_channel(host, port, channel_creds) - return stub_factory(channel) diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py deleted file mode 100644 index d5670ec..0000000 --- a/gcloud_bigtable/cluster.py +++ /dev/null @@ -1,482 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Cluster.""" - - -import re - -from google.longrunning import operations_pb2 - -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as table_messages_pb2) -from gcloud_bigtable._non_upstream_helpers import _pb_timestamp_to_datetime -from gcloud_bigtable.table import Table - - -_CLUSTER_NAME_RE = re.compile(r'^projects/(?P[^/]+)/' - r'zones/(?P[^/]+)/clusters/' - r'(?P[a-z][-a-z0-9]*)$') -_OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/zones/([^/]+)/' - r'clusters/([a-z][-a-z0-9]*)/operations/' - r'(?P\d+)$') -_DEFAULT_SERVE_NODES = 3 -_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' -_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' -_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' -_UPDATE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata' -_UNDELETE_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata' -_TYPE_URL_MAP = { - _CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata, - _UPDATE_CREATE_METADATA: messages_pb2.UpdateClusterMetadata, - _UNDELETE_CREATE_METADATA: messages_pb2.UndeleteClusterMetadata, -} - - -def _prepare_create_request(cluster): - """Creates a protobuf request for a CreateCluster request. - - :type cluster: :class:`Cluster` - :param cluster: The cluster to be created. - - :rtype: :class:`.messages_pb2.CreateClusterRequest` - :returns: The CreateCluster request object containing the cluster info. - """ - zone_full_name = ('projects/' + cluster._client.project + - '/zones/' + cluster.zone) - return messages_pb2.CreateClusterRequest( - name=zone_full_name, - cluster_id=cluster.cluster_id, - cluster=data_pb2.Cluster( - display_name=cluster.display_name, - serve_nodes=cluster.serve_nodes, - ), - ) - - -def _parse_pb_any_to_native(any_val, expected_type=None): - """Convert a serialized "google.protobuf.Any" value to actual type. - - :type any_val: :class:`google.protobuf.any_pb2.Any` - :param any_val: A serialized protobuf value container. - - :type expected_type: str - :param expected_type: (Optional) The type URL we expect ``any_val`` - to have. - - :rtype: object - :returns: The de-serialized object. - :raises: :class:`ValueError ` if the - ``expected_type`` does not match the ``type_url`` on the input. - """ - if expected_type is not None and expected_type != any_val.type_url: - raise ValueError('Expected type: %s, Received: %s' % ( - expected_type, any_val.type_url)) - container_class = _TYPE_URL_MAP[any_val.type_url] - return container_class.FromString(any_val.value) - - -def _process_operation(operation_pb): - """Processes a create protobuf response. - - :type operation_pb: :class:`google.longrunning.operations_pb2.Operation` - :param operation_pb: The long-running operation response from a - Create/Update/Undelete cluster request. - - :rtype: tuple - :returns: A pair of an integer and datetime stamp. The integer is the ID - of the operation (``operation_id``) and the timestamp when - the create operation began (``operation_begin``). - :raises: :class:`ValueError ` if the operation name - doesn't match the :data:`_OPERATION_NAME_RE` regex. - """ - match = _OPERATION_NAME_RE.match(operation_pb.name) - if match is None: - raise ValueError('Operation name was not in the expected ' - 'format after a cluster modification.', - operation_pb.name) - operation_id = int(match.group('operation_id')) - - request_metadata = _parse_pb_any_to_native(operation_pb.metadata) - operation_begin = _pb_timestamp_to_datetime( - request_metadata.request_time) - - return operation_id, operation_begin - - -class Operation(object): - """Representation of a Google API Long-Running Operation. - - In particular, these will be the result of operations on - clusters using the Cloud Bigtable API. - - :type op_type: str - :param op_type: The type of operation being performed. Expect - ``create``, ``update`` or ``undelete``. - - :type op_id: int - :param op_id: The ID of the operation. - - :type begin: :class:`datetime.datetime` - :param begin: The time when the operation was started. - - :type cluster: :class:`Cluster` - :param cluster: The cluster that created the operation. - """ - - def __init__(self, op_type, op_id, begin, cluster=None): - self.op_type = op_type - self.op_id = op_id - self.begin = begin - self._cluster = cluster - self._complete = False - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.op_type == self.op_type and - other.op_id == self.op_id and - other.begin == self.begin and - other._cluster == self._cluster and - other._complete == self._complete) - - def __ne__(self, other): - return not self.__eq__(other) - - def finished(self): - """Check if the operation has finished. - - :rtype: bool - :returns: A boolean indicating if the current operation has completed. - :raises: :class:`ValueError ` if the operation - has already completed. - """ - if self._complete: - raise ValueError('The operation has completed.') - - operation_name = ('operations/' + self._cluster.name + - '/operations/%d' % (self.op_id,)) - request_pb = operations_pb2.GetOperationRequest(name=operation_name) - # We expect a `google.longrunning.operations_pb2.Operation`. - operation_pb = self._cluster._client._operations_stub.GetOperation( - request_pb, self._cluster._client.timeout_seconds) - - if operation_pb.done: - self._complete = True - return True - else: - return False - - -class Cluster(object): - """Representation of a Google Cloud Bigtable Cluster. - - We can use a :class:`Cluster` to: - - * :meth:`reload` itself - * :meth:`create` itself - * :meth:`update` itself - * :meth:`delete` itself - * :meth:`undelete` itself - - .. note:: - - For now, we leave out the ``default_storage_type`` (an enum) - which if not sent will end up as :data:`.data_pb2.STORAGE_SSD`. - - :type zone: str - :param zone: The name of the zone where the cluster resides. - - :type cluster_id: str - :param cluster_id: The ID of the cluster. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides - authorization and a project ID. - - :type display_name: str - :param display_name: (Optional) The display name for the cluster in the - Cloud Console UI. (Must be between 4 and 30 - characters.) If this value is not set in the - constructor, will fall back to the cluster ID. - - :type serve_nodes: int - :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3 (``_DEFAULT_SERVE_NODES``). - """ - - def __init__(self, zone, cluster_id, client, - display_name=None, serve_nodes=_DEFAULT_SERVE_NODES): - self.zone = zone - self.cluster_id = cluster_id - self.display_name = display_name or cluster_id - self.serve_nodes = serve_nodes - self._client = client - - def table(self, table_id): - """Factory to create a table associated with this cluster. - - :type table_id: str - :param table_id: The ID of the table. - - :rtype: :class:`Table ` - :returns: The table owned by this cluster. - """ - return Table(table_id, self) - - def _update_from_pb(self, cluster_pb): - if not cluster_pb.display_name: # Simple field (string) - raise ValueError('Cluster protobuf does not contain display_name') - if not cluster_pb.serve_nodes: # Simple field (int32) - raise ValueError('Cluster protobuf does not contain serve_nodes') - self.display_name = cluster_pb.display_name - self.serve_nodes = cluster_pb.serve_nodes - - @classmethod - def from_pb(cls, cluster_pb, client): - """Creates a cluster instance from a protobuf. - - :type cluster_pb: :class:`bigtable_cluster_data_pb2.Cluster` - :param cluster_pb: A cluster protobuf object. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. - - :rtype: :class:`Cluster` - :returns: The cluster parsed from the protobuf response. - :raises: :class:`ValueError ` if the cluster - name does not match :data:`_CLUSTER_NAME_RE` or if the parsed - project ID does not match the project ID on the client. - """ - match = _CLUSTER_NAME_RE.match(cluster_pb.name) - if match is None: - raise ValueError('Cluster protobuf name was not in the ' - 'expected format.', cluster_pb.name) - if match.group('project') != client.project: - raise ValueError('Project ID on cluster does not match the ' - 'project ID on the client') - - result = cls(match.group('zone'), match.group('cluster_id'), client) - result._update_from_pb(cluster_pb) - return result - - def copy(self): - """Make a copy of this cluster. - - Copies the local data stored as simple types but does not copy the - current state of any operations with the Cloud Bigtable API. Also - copies the client attached to this instance. - - :rtype: :class:`.Cluster` - :returns: A copy of the current cluster. - """ - new_client = self._client.copy() - return self.__class__(self.zone, self.cluster_id, new_client, - display_name=self.display_name, - serve_nodes=self.serve_nodes) - - @property - def name(self): - """Cluster name used in requests. - - .. note:: - This property will not change if ``zone`` and ``cluster_id`` do not, - but the return value is not cached. - - The cluster name is of the form - - ``"projects/{project}/zones/{zone}/clusters/{cluster_id}"`` - - :rtype: str - :returns: The cluster name. - """ - return (self._client.project_name + '/zones/' + self.zone + - '/clusters/' + self.cluster_id) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - # NOTE: This does not compare the configuration values, such as - # the serve_nodes or display_name. Instead, it only compares - # identifying values zone, cluster ID and client. This is - # intentional, since the same cluster can be in different states - # if not synchronized. Clusters with similar zone/cluster - # settings but different clients can't be used in the same way. - return (other.zone == self.zone and - other.cluster_id == self.cluster_id and - other._client == self._client) - - def __ne__(self, other): - return not self.__eq__(other) - - def reload(self): - """Reload the metadata for this cluster.""" - request_pb = messages_pb2.GetClusterRequest(name=self.name) - # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = self._client._cluster_stub.GetCluster( - request_pb, self._client.timeout_seconds) - - # NOTE: _update_from_pb does not check that the project, zone and - # cluster ID on the response match the request. - self._update_from_pb(cluster_pb) - - def create(self): - """Create this cluster. - - .. note:: - - Uses the ``project``, ``zone`` and ``cluster_id`` on the current - :class:`Cluster` in addition to the ``display_name`` and - ``serve_nodes``. If you'd like to change them before creating, - reset the values via - - .. code:: python - - cluster.display_name = 'New display name' - cluster.cluster_id = 'i-changed-my-mind' - - before calling :meth:`create`. - - :rtype: :class:`Operation` - :returns: The long-running operation corresponding to the - create operation. - """ - request_pb = _prepare_create_request(self) - # We expect a `google.longrunning.operations_pb2.Operation`. - cluster_pb = self._client._cluster_stub.CreateCluster( - request_pb, self._client.timeout_seconds) - - op_id, op_begin = _process_operation(cluster_pb.current_operation) - return Operation('create', op_id, op_begin, cluster=self) - - def update(self): - """Update this cluster. - - .. note:: - - Updates the ``display_name`` and ``serve_nodes``. If you'd like to - change them before updating, reset the values via - - .. code:: python - - cluster.display_name = 'New display name' - cluster.serve_nodes = 3 - - before calling :meth:`update`. - - :rtype: :class:`Operation` - :returns: The long-running operation corresponding to the - update operation. - """ - request_pb = data_pb2.Cluster( - name=self.name, - display_name=self.display_name, - serve_nodes=self.serve_nodes, - ) - # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = self._client._cluster_stub.UpdateCluster( - request_pb, self._client.timeout_seconds) - - op_id, op_begin = _process_operation(cluster_pb.current_operation) - return Operation('update', op_id, op_begin, cluster=self) - - def delete(self): - """Delete this cluster. - - Marks a cluster and all of its tables for permanent deletion in 7 days. - - Immediately upon completion of the request: - - * Billing will cease for all of the cluster's reserved resources. - * The cluster's ``delete_time`` field will be set 7 days in the future. - - Soon afterward: - - * All tables within the cluster will become unavailable. - - Prior to the cluster's ``delete_time``: - - * The cluster can be recovered with a call to ``UndeleteCluster``. - * All other attempts to modify or delete the cluster will be rejected. - - At the cluster's ``delete_time``: - - * The cluster and **all of its tables** will immediately and - irrevocably disappear from the API, and their data will be - permanently deleted. - """ - request_pb = messages_pb2.DeleteClusterRequest(name=self.name) - # We expect a `google.protobuf.empty_pb2.Empty` - self._client._cluster_stub.DeleteCluster( - request_pb, self._client.timeout_seconds) - - def undelete(self): - """Undelete this cluster. - - Cancels the scheduled deletion of an cluster and begins preparing it to - resume serving. The returned operation will also be embedded as the - cluster's ``current_operation``. - - Immediately upon completion of this request: - - * The cluster's ``delete_time`` field will be unset, protecting it from - automatic deletion. - - Until completion of the returned operation: - - * The operation cannot be cancelled. - - Upon completion of the returned operation: - - * Billing for the cluster's resources will resume. - * All tables within the cluster will be available. - - :rtype: :class:`Operation` - :returns: The long-running operation corresponding to the - undelete operation. - """ - request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) - # We expect a `google.longrunning.operations_pb2.Operation`. - operation_pb2 = self._client._cluster_stub.UndeleteCluster( - request_pb, self._client.timeout_seconds) - - op_id, op_begin = _process_operation(operation_pb2) - return Operation('undelete', op_id, op_begin, cluster=self) - - def list_tables(self): - """List the tables in this cluster. - - :rtype: list of :class:`Table ` - :returns: The list of tables owned by the cluster. - :raises: :class:`ValueError ` if one of the - returned tables has a name that is not of the expected format. - """ - request_pb = table_messages_pb2.ListTablesRequest(name=self.name) - # We expect a `table_messages_pb2.ListTablesResponse` - table_list_pb = self._client._table_stub.ListTables( - request_pb, self._client.timeout_seconds) - - result = [] - for table_pb in table_list_pb.tables: - table_prefix = self.name + '/tables/' - if not table_pb.name.startswith(table_prefix): - raise ValueError('Table name %s not of expected format' % ( - table_pb.name,)) - table_id = table_pb.name[len(table_prefix):] - result.append(self.table(table_id)) - - return result diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py deleted file mode 100644 index dad80d3..0000000 --- a/gcloud_bigtable/column_family.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Column Family.""" - - -import datetime - -from google.protobuf import duration_pb2 - -from gcloud_bigtable._generated import bigtable_table_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._non_upstream_helpers import _total_seconds - - -def _timedelta_to_duration_pb(timedelta_val): - """Convert a Python timedelta object to a duration protobuf. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type timedelta_val: :class:`datetime.timedelta` - :param timedelta_val: A timedelta object. - - :rtype: :class:`google.protobuf.duration_pb2.Duration` - :returns: A duration object equivalent to the time delta. - """ - seconds_decimal = _total_seconds(timedelta_val) - # Truncate the parts other than the integer. - seconds = int(seconds_decimal) - if seconds_decimal < 0: - signed_micros = timedelta_val.microseconds - 10**6 - else: - signed_micros = timedelta_val.microseconds - # Convert nanoseconds to microseconds. - nanos = 1000 * signed_micros - return duration_pb2.Duration(seconds=seconds, nanos=nanos) - - -def _duration_pb_to_timedelta(duration_pb): - """Convert a duration protobuf to a Python timedelta object. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type duration_pb: :class:`google.protobuf.duration_pb2.Duration` - :param duration_pb: A protobuf duration object. - - :rtype: :class:`datetime.timedelta` - :returns: The converted timedelta object. - """ - return datetime.timedelta( - seconds=duration_pb.seconds, - microseconds=(duration_pb.nanos / 1000.0), - ) - - -class GarbageCollectionRule(object): - """Garbage collection rule for column families within a table. - - Cells in the column family (within a table) fitting the rule will be - deleted during garbage collection. - - .. note:: - - This class is a do-nothing base class for all GC rules. - - .. note:: - - A string ``gc_expression`` can also be used with API requests, but - that value would be superceded by a ``gc_rule``. As a result, we - don't support that feature and instead support via native classes. - """ - - def __ne__(self, other): - return not self.__eq__(other) - - -class MaxVersionsGCRule(GarbageCollectionRule): - """Garbage collection limiting the number of versions of a cell. - - :type max_num_versions: int - :param max_num_versions: The maximum number of versions - """ - - def __init__(self, max_num_versions): - self.max_num_versions = max_num_versions - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.max_num_versions == self.max_num_versions - - def to_pb(self): - """Converts the garbage collection rule to a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - return data_pb2.GcRule(max_num_versions=self.max_num_versions) - - -class MaxAgeGCRule(GarbageCollectionRule): - """Garbage collection limiting the age of a cell. - - :type max_age: :class:`datetime.timedelta` - :param max_age: The maximum age allowed for a cell in the table. - """ - - def __init__(self, max_age): - self.max_age = max_age - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.max_age == self.max_age - - def to_pb(self): - """Converts the garbage collection rule to a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - max_age = _timedelta_to_duration_pb(self.max_age) - return data_pb2.GcRule(max_age=max_age) - - -class GCRuleUnion(GarbageCollectionRule): - """Union of garbage collection rules. - - :type rules: list - :param rules: List of :class:`GarbageCollectionRule`. - """ - - def __init__(self, rules): - self.rules = rules - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.rules == self.rules - - def to_pb(self): - """Converts the union into a single gc rule as a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - union = data_pb2.GcRule.Union( - rules=[rule.to_pb() for rule in self.rules]) - return data_pb2.GcRule(union=union) - - -class GCRuleIntersection(GarbageCollectionRule): - """Intersection of garbage collection rules. - - :type rules: list - :param rules: List of :class:`GarbageCollectionRule`. - """ - - def __init__(self, rules): - self.rules = rules - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.rules == self.rules - - def to_pb(self): - """Converts the intersection into a single gc rule as a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - intersection = data_pb2.GcRule.Intersection( - rules=[rule.to_pb() for rule in self.rules]) - return data_pb2.GcRule(intersection=intersection) - - -class ColumnFamily(object): - """Representation of a Google Cloud Bigtable Column Family. - - We can use a :class:`ColumnFamily` to: - - * :meth:`create` itself - * :meth:`update` itself - * :meth:`delete` itself - - :type column_family_id: str - :param column_family_id: The ID of the column family. Must be of the - form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type table: :class:`Table ` - :param table: The table that owns the column family. - - :type gc_rule: :class:`GarbageCollectionRule` - :param gc_rule: (Optional) The garbage collection settings for this - column family. - """ - - def __init__(self, column_family_id, table, gc_rule=None): - self.column_family_id = column_family_id - self._table = table - self.gc_rule = gc_rule - - @property - def name(self): - """Column family name used in requests. - - .. note:: - - This property will not change if ``column_family_id`` does not, but - the return value is not cached. - - The table name is of the form - - ``"projects/../zones/../clusters/../tables/../columnFamilies/.."`` - - :rtype: str - :returns: The column family name. - """ - return self._table.name + '/columnFamilies/' + self.column_family_id - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other._table == self._table and - other.gc_rule == self.gc_rule) - - def __ne__(self, other): - return not self.__eq__(other) - - def create(self): - """Create this column family.""" - if self.gc_rule is None: - column_family = data_pb2.ColumnFamily() - else: - column_family = data_pb2.ColumnFamily(gc_rule=self.gc_rule.to_pb()) - request_pb = messages_pb2.CreateColumnFamilyRequest( - name=self._table.name, - column_family_id=self.column_family_id, - column_family=column_family, - ) - client = self._table._cluster._client - # We expect a `.data_pb2.ColumnFamily`. We ignore it since the only - # data it contains are the GC rule and the column family ID already - # stored on this instance. - client._table_stub.CreateColumnFamily(request_pb, - client.timeout_seconds) - - def update(self): - """Update this column family. - - .. note:: - - Only the GC rule can be updated. By changing the column family ID, - you will simply be referring to a different column family. - """ - request_kwargs = {'name': self.name} - if self.gc_rule is not None: - request_kwargs['gc_rule'] = self.gc_rule.to_pb() - request_pb = data_pb2.ColumnFamily(**request_kwargs) - client = self._table._cluster._client - # We expect a `.data_pb2.ColumnFamily`. We ignore it since the only - # data it contains are the GC rule and the column family ID already - # stored on this instance. - client._table_stub.UpdateColumnFamily(request_pb, - client.timeout_seconds) - - def delete(self): - """Delete this column family.""" - request_pb = messages_pb2.DeleteColumnFamilyRequest(name=self.name) - client = self._table._cluster._client - # We expect a `google.protobuf.empty_pb2.Empty` - client._table_stub.DeleteColumnFamily(request_pb, - client.timeout_seconds) - - -def _gc_rule_from_pb(gc_rule_pb): - """Convert a protobuf GC rule to a native object. - - :type gc_rule_pb: :class:`.data_pb2.GcRule` - :param gc_rule_pb: The GC rule to convert. - - :rtype: :class:`GarbageCollectionRule` or :data:`NoneType ` - :returns: An instance of one of the native rules defined - in :module:`column_family` or :data:`None` if no values were - set on the protobuf passed in. - :raises: :class:`ValueError ` if the rule name - is unexpected. - """ - rule_name = gc_rule_pb.WhichOneof('rule') - if rule_name is None: - return None - - if rule_name == 'max_num_versions': - return MaxVersionsGCRule(gc_rule_pb.max_num_versions) - elif rule_name == 'max_age': - max_age = _duration_pb_to_timedelta(gc_rule_pb.max_age) - return MaxAgeGCRule(max_age) - elif rule_name == 'union': - return GCRuleUnion([_gc_rule_from_pb(rule) - for rule in gc_rule_pb.union.rules]) - elif rule_name == 'intersection': - rules = [_gc_rule_from_pb(rule) - for rule in gc_rule_pb.intersection.rules] - return GCRuleIntersection(rules) - else: - raise ValueError('Unexpected rule name', rule_name) diff --git a/gcloud_bigtable/happybase/__init__.py b/gcloud_bigtable/happybase/__init__.py deleted file mode 100644 index eb274a3..0000000 --- a/gcloud_bigtable/happybase/__init__.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase package. - -This package is intended to emulate the HappyBase library using -Google Cloud Bigtable as the backing store. - -Differences in Public API -------------------------- - -Some concepts from HBase/Thrift do not map directly to the Cloud -Bigtable API. As a result, the following instance methods and functions -could not be implemented: - -* :meth:`.Connection.enable_table` - no concept of enabled/disabled -* :meth:`.Connection.disable_table` - no concept of enabled/disabled -* :meth:`.Connection.is_table_enabled` - no concept of enabled/disabled -* :meth:`.Connection.compact_table` - table storage is opaque to user -* :func:`make_row() ` - helper - needed for Thrift library -* :func:`make_ordered_row() ` - - helper needed for Thrift library -* :meth:`Table.regions() ` - - tables in Cloud Bigtable do not expose internal storage details -* :meth:`Table.counter_set() \ - ` - method can't - be atomic, so we disable it -* The ``__version__`` value for the HappyBase package is :data:`None` - -In addition, many of the constants from :mod:`.connection` are specific -to HBase and are defined as :data:`None` in our module: - -* ``COMPAT_MODES`` -* ``THRIFT_TRANSPORTS`` -* ``THRIFT_PROTOCOLS`` -* ``DEFAULT_HOST`` -* ``DEFAULT_PORT`` -* ``DEFAULT_TRANSPORT`` -* ``DEFAULT_COMPAT`` -* ``DEFAULT_PROTOCOL`` - -Two of these ``DEFAULT_HOST`` and ``DEFAULT_PORT``, are even imported in -the main ``happybase`` package. - -Finally, we do not provide the ``util`` module. Though it is public in the -HappyBase library, it provides no core functionality. - -API Behavior Changes --------------------- - -* Since there is no concept of an enabled / disabled table, calling - :meth:`.Connection.delete_table` with ``disable=True`` can't be supported. - Using that argument will result in a - :class:`ValueError `. -* The :class:`.Connection` constructor **disables** the use of several - arguments and will throw a :class:`ValueError ` if - any of them are passed in as keyword arguments. The arguments are: - - * ``host`` - * ``port`` - * ``compat`` - * ``transport`` - * ``protocol`` -* In order to make :class:`.Connection` compatible with Cloud Bigtable, we - add a ``cluster`` keyword argument to allow user's to pass in their own - :class:`.Cluster` (which they can construct beforehand). - - For example: - - .. code:: python - - from gcloud_bigtable.client import Client - client = Client(project=PROJECT_ID, admin=True) - cluster = client.cluster(zone, cluster_id) - cluster.reload() - - from gcloud_bigtable.happybase import Connection - connection = Connection(cluster=cluster) - -* Any uses of the ``wal`` (Write Ahead Log) argument will result in a - :class:`ValueError ` as well. This includes - uses in: - - * :class:`.Batch` constructor - * :meth:`.Batch.put` - * :meth:`.Batch.delete` - * :meth:`Table.put() ` - * :meth:`Table.delete() ` - * :meth:`Table.batch() ` factory -* When calling :meth:`.Connection.create_table`, the majority of HBase column - family options cannot be used. Among - - * ``max_versions`` - * ``compression`` - * ``in_memory`` - * ``bloom_filter_type`` - * ``bloom_filter_vector_size`` - * ``bloom_filter_nb_hashes`` - * ``block_cache_enabled`` - * ``time_to_live`` - - Only ``max_versions`` and ``time_to_live`` are availabe in Cloud Bigtable - (as ``max_num_versions`` and ``max_age``). - - In addition to using a dictionary for specifying column family options, - we also accept instances of :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion` or - :class:`.GarbageCollectionRuleIntersection`. -* The ``batch_size`` attribute in :class:`.Batch` cannot be truly mapped - from HBase to Cloud Bigtable. The main reason for this is that the Cloud - Bigtable API can only send mutations for a single row (via ``MutateRow``, - ``CheckAndMutateRow``, and ``ReadModifyWriteRow``) whereas HBase sends - all mutations at once. This requires a single request to be sent for each - mutated row in the batch. This should not be noticeable since gRPC - uses HTTP/2. However, some of the requests may fail part way through and - the process of applying all mutations cannot be rolled back. -* :meth:`Table.scan() ` no longer - accepts the following arguments (which will result in a - :class:`ValueError `): - - * ``batch_size`` - * ``scan_batching`` - * ``sorted_columns`` - -* Using a HBase filter string in - :meth:`Table.scan() ` is - not possible with Cloud Bigtable and will result in a - :class:`TypeError `. However, the method now accepts - :class:`.RowFilter` instances (and related classes). -* :meth:`.Batch.delete` (and hence - :meth:`Table.delete() `) - will fail when either a row or column family delete is attempted with - a ``timestamp``. This is because the Cloud Bigtable API uses the - ``DeleteFromFamily`` and ``DeleteFromRow`` mutations for these deletes, and - neither of these mutations support a timestamp. -""" - -from gcloud_bigtable.happybase.batch import Batch -from gcloud_bigtable.happybase.connection import Connection -from gcloud_bigtable.happybase.connection import DEFAULT_HOST -from gcloud_bigtable.happybase.connection import DEFAULT_PORT -from gcloud_bigtable.happybase.pool import ConnectionPool -from gcloud_bigtable.happybase.pool import NoConnectionsAvailable -from gcloud_bigtable.happybase.table import Table - - -# Values from HappyBase that we don't reproduce / are not relevant. -__version__ = None diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py deleted file mode 100644 index 333cd06..0000000 --- a/gcloud_bigtable/happybase/batch.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase batch module.""" - - -import datetime -import warnings - -import six - -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -from gcloud_bigtable.row import TimestampRange - - -# BEGIN: Renames for upstream. -# pylint: disable=invalid-name -_datetime_from_microseconds = _microseconds_to_timestamp -# pylint: enable=invalid-name -# END: Renames for upstream. -_WAL_SENTINEL = object() -# Assumed granularity of timestamps in Cloud Bigtable. -_ONE_MILLISECOND = datetime.timedelta(microseconds=1000) -_WARN = warnings.warn -_WAL_WARNING = ('The wal argument (Write-Ahead-Log) is not ' - 'supported by Cloud Bigtable.') - - -class Batch(object): - """Batch class for accumulating mutations. - - .. note:: - - When using a batch with ``transaction=False`` as a context manager - (i.e. in a ``with`` statement), mutations will still be sent as - row mutations even if the context manager exits with an error. - This behavior is in place to match the behavior in the HappyBase - HBase / Thrift implementation. - - :type table: :class:`Table ` - :param table: The table where mutations will be applied. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the epoch) - that all mutations will be applied at. - - :type batch_size: int - :param batch_size: (Optional) The maximum number of mutations to allow - to accumulate before committing them. - - :type transaction: bool - :param transaction: Flag indicating if the mutations should be sent - transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. - - :type wal: object - :param wal: Unused parameter (Boolean for using the HBase Write Ahead Log). - Provided for compatibility with HappyBase, but irrelevant for - Cloud Bigtable since it does not have a Write Ahead Log. - - :raises: :class:`TypeError ` if ``batch_size`` - is set and ``transaction=True``. - :class:`ValueError ` if ``batch_size`` - is not positive. - """ - - def __init__(self, table, timestamp=None, batch_size=None, - transaction=False, wal=_WAL_SENTINEL): - if wal is not _WAL_SENTINEL: - _WARN(_WAL_WARNING) - - if batch_size is not None: - if transaction: - raise TypeError('When batch_size is set, a Batch cannot be ' - 'transactional') - if batch_size <= 0: - raise ValueError('batch_size must be positive') - - self._table = table - self._batch_size = batch_size - self._timestamp = self._delete_range = None - - # Timestamp is in milliseconds, convert to microseconds. - if timestamp is not None: - self._timestamp = _datetime_from_microseconds(1000 * timestamp) - # For deletes, we get the very next timestamp (assuming timestamp - # granularity is milliseconds). This is because HappyBase users - # expect HBase deletes to go **up to** and **including** the - # timestamp while Cloud Bigtable Time Ranges **exclude** the - # final timestamp. - next_timestamp = self._timestamp + _ONE_MILLISECOND - self._delete_range = TimestampRange(end=next_timestamp) - - self._transaction = transaction - - # Internal state for tracking mutations. - self._row_map = {} - self._mutation_count = 0 - - def send(self): - """Send / commit the batch of mutations to the server.""" - for row in self._row_map.values(): - # commit() does nothing if row hasn't accumulated any mutations. - row.commit() - - self._row_map.clear() - self._mutation_count = 0 - - def _try_send(self): - """Send / commit the batch if mutations have exceeded batch size.""" - if self._batch_size and self._mutation_count >= self._batch_size: - self.send() - - def _get_row(self, row_key): - """Gets a row that will hold mutations. - - If the row is not already cached on the current batch, a new row will - be created. - - :type row_key: str - :param row_key: The row key for a row stored in the map. - - :rtype: :class:`Row ` - :returns: The newly created or stored row that will hold mutations. - """ - if row_key not in self._row_map: - table = self._table._low_level_table - self._row_map[row_key] = table.row(row_key) - - return self._row_map[row_key] - - def put(self, row, data, wal=_WAL_SENTINEL): - """Insert data into a row in the table owned by this batch. - - :type row: str - :param row: The row key where the mutation will be "put". - - :type data: dict - :param data: Dictionary containing the data to be inserted. The keys - are columns names (of the form ``fam:col``) and the values - are strings (bytes) to be stored in those columns. - - :type wal: object - :param wal: Unused parameter (to over-ride the default on the - instance). Provided for compatibility with HappyBase, but - irrelevant for Cloud Bigtable since it does not have a - Write Ahead Log. - """ - if wal is not _WAL_SENTINEL: - _WARN(_WAL_WARNING) - - row_object = self._get_row(row) - # Make sure all the keys are valid before beginning - # to add mutations. - column_pairs = _get_column_pairs(six.iterkeys(data), - require_qualifier=True) - for column_family_id, column_qualifier in column_pairs: - value = data[column_family_id + ':' + column_qualifier] - row_object.set_cell(column_family_id, column_qualifier, - value, timestamp=self._timestamp) - - self._mutation_count += len(data) - self._try_send() - - def _delete_columns(self, columns, row_object): - """Adds delete mutations for a list of columns and column families. - - :type columns: list - :param columns: Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type row_object: :class:`Row ` - :param row_object: The row which will hold the delete mutations. - - :raises: :class:`ValueError ` if the delete - timestamp range is set on the current batch, but a - column family delete is attempted. - """ - column_pairs = _get_column_pairs(columns) - for column_family_id, column_qualifier in column_pairs: - if column_qualifier is None: - if self._delete_range is not None: - raise ValueError('The Cloud Bigtable API does not support ' - 'adding a timestamp to ' - '"DeleteFromFamily" ') - row_object.delete_cells(column_family_id, - columns=row_object.ALL_COLUMNS) - else: - row_object.delete_cell(column_family_id, - column_qualifier, - time_range=self._delete_range) - - def delete(self, row, columns=None, wal=_WAL_SENTINEL): - """Delete data from a row in the table owned by this batch. - - :type row: str - :param row: The row key where the delete will occur. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - If not used, will delete the entire row. - - :type wal: object - :param wal: Unused parameter (to over-ride the default on the - instance). Provided for compatibility with HappyBase, but - irrelevant for Cloud Bigtable since it does not have a - Write Ahead Log. - - :raises: If if the delete timestamp range is set on the - current batch, but a full row delete is attempted. - """ - if wal is not _WAL_SENTINEL: - _WARN(_WAL_WARNING) - - row_object = self._get_row(row) - - if columns is None: - # Delete entire row. - if self._delete_range is not None: - raise ValueError('The Cloud Bigtable API does not support ' - 'adding a timestamp to "DeleteFromRow" ' - 'mutations') - row_object.delete() - self._mutation_count += 1 - else: - self._delete_columns(columns, row_object) - self._mutation_count += len(columns) - - self._try_send() - - def __enter__(self): - """Enter context manager, no set-up required.""" - return self - - def __exit__(self, exc_type, exc_value, traceback): - """Exit context manager, no set-up required. - - :type exc_type: type - :param exc_type: The type of the exception if one occurred while the - context manager was active. Otherwise, :data:`None`. - - :type exc_value: :class:`Exception ` - :param exc_value: An instance of ``exc_type`` if an exception occurred - while the context was active. - Otherwise, :data:`None`. - - :type traceback: ``traceback`` type - :param traceback: The traceback where the exception occurred (if one - did occur). Otherwise, :data:`None`. - """ - # If the context manager encountered an exception and the batch is - # transactional, we don't commit the mutations. - if self._transaction and exc_type is not None: - return - - # NOTE: For non-transactional batches, this will even commit mutations - # if an error occurred during the context manager. - self.send() - - -def _get_column_pairs(columns, require_qualifier=False): - """Turns a list of column or column families into parsed pairs. - - Turns a column family (``fam`` or ``fam:``) into a pair such - as ``['fam', None]`` and turns a column (``fam:col``) into - ``['fam', 'col']``. - - :type columns: list - :param columns: Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type require_qualifier: bool - :param require_qualifier: Boolean indicating if the columns should - all have a qualifier or not. - - :rtype: list - :returns: List of pairs, where the first element in each pair is the - column family and the second is the column qualifier - (or :data:`None`). - :raises: :class:`ValueError ` if any of the columns - are not of the expected format. - :class:`ValueError ` if - ``require_qualifier`` is :data:`True` and one of the values is - for an entire column family - """ - column_pairs = [] - for column in columns: - if isinstance(column, six.binary_type): - column = column.decode('utf-8') - # Remove trailing colons (i.e. for standalone column family). - if column.endswith(u':'): - column = column[:-1] - num_colons = column.count(u':') - if num_colons == 0: - # column is a column family. - if require_qualifier: - raise ValueError('column does not contain a qualifier', - column) - else: - column_pairs.append([column, None]) - elif num_colons == 1: - column_pairs.append(column.split(u':')) - else: - raise ValueError('Column contains the : separator more than once') - - return column_pairs diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py deleted file mode 100644 index e82f600..0000000 --- a/gcloud_bigtable/happybase/connection.py +++ /dev/null @@ -1,459 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase connection module.""" - - -import datetime -import warnings - -import six - -from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import GCRuleIntersection -from gcloud_bigtable.column_family import MaxAgeGCRule -from gcloud_bigtable.column_family import MaxVersionsGCRule -from gcloud_bigtable.happybase.table import Table -from gcloud_bigtable.table import Table as _LowLevelTable - - -# Constants reproduced here for HappyBase compatibility, though values -# are all null. -COMPAT_MODES = None -THRIFT_TRANSPORTS = None -THRIFT_PROTOCOLS = None -DEFAULT_HOST = None -DEFAULT_PORT = None -DEFAULT_TRANSPORT = None -DEFAULT_COMPAT = None -DEFAULT_PROTOCOL = None - -_LEGACY_ARGS = frozenset(('host', 'port', 'compat', 'transport', 'protocol')) -_WARN = warnings.warn - - -def _get_cluster(timeout=None): - """Gets cluster for the default project. - - Creates a client with the inferred credentials and project ID from - the local environment. Then uses :meth:`.Client.list_clusters` to - get the unique cluster owned by the project. - - If the request fails for any reason, or if there isn't exactly one cluster - owned by the project, then this function will fail. - - :type timeout: int - :param timeout: (Optional) The socket timeout in milliseconds. - - :rtype: :class:`gcloud_bigtable.cluster.Cluster` - :returns: The unique cluster owned by the project inferred from - the environment. - :raises: :class:`ValueError ` if there is a failed - zone or any number of clusters other than one. - """ - client_kwargs = {'admin': True} - if timeout is not None: - client_kwargs['timeout_seconds'] = timeout / 1000.0 - client = Client(**client_kwargs) - try: - client.start() - clusters, failed_zones = client.list_clusters() - finally: - client.stop() - - if len(failed_zones) != 0: - raise ValueError('Determining cluster via ListClusters encountered ' - 'failed zones.') - if len(clusters) == 0: - raise ValueError('This client doesn\'t have access to any clusters.') - if len(clusters) > 1: - raise ValueError('This client has access to more than one cluster. ' - 'Please directly pass the cluster you\'d ' - 'like to use.') - return clusters[0] - - -class Connection(object): - """Connection to Cloud Bigtable backend. - - .. note:: - - If you pass a ``cluster``, it will be :meth:`.Cluster.copy`-ed before - being stored on the new connection. This also copies the - :class:`.Client` that created the :class:`.Cluster` instance and the - :class:`Credentials ` stored on the - client. - - The arguments ``host``, ``port``, ``compat``, ``transport`` and - ``protocol`` are allowed (as keyword arguments) for compatibility with - HappyBase. However, they will not be used in anyway, and will cause a - warning if passed. - - :type timeout: int - :param timeout: (Optional) The socket timeout in milliseconds. - - :type autoconnect: bool - :param autoconnect: (Optional) Whether the connection should be - :meth:`open`-ed during construction. - - :type table_prefix: str - :param table_prefix: (Optional) Prefix used to construct table names. - - :type table_prefix_separator: str - :param table_prefix_separator: (Optional) Separator used with - ``table_prefix``. Defaults to ``_``. - - :type cluster: :class:`gcloud_bigtable.cluster.Cluster` - :param cluster: (Optional) A Cloud Bigtable cluster. The instance also - owns a client for making gRPC requests to the Cloud - Bigtable API. If not passed in, defaults to creating client - with ``admin=True`` and using the ``timeout`` here for the - ``timeout_seconds`` argument to the :class:`.Client`` - constructor. The credentials for the client - will be the implicit ones loaded from the environment. - Then that client is used to retrieve all the clusters - owned by the client's project. - - :type kwargs: dict - :param kwargs: Remaining keyword arguments. Provided for HappyBase - compatibility. - """ - - _cluster = None - - def __init__(self, timeout=None, autoconnect=True, table_prefix=None, - table_prefix_separator='_', cluster=None, **kwargs): - self._handle_legacy_args(kwargs) - if table_prefix is not None: - if not isinstance(table_prefix, six.string_types): - raise TypeError('table_prefix must be a string', 'received', - table_prefix, type(table_prefix)) - - if not isinstance(table_prefix_separator, six.string_types): - raise TypeError('table_prefix_separator must be a string', - 'received', table_prefix_separator, - type(table_prefix_separator)) - - self.table_prefix = table_prefix - self.table_prefix_separator = table_prefix_separator - - if cluster is None: - self._cluster = _get_cluster(timeout=timeout) - else: - if timeout is not None: - raise ValueError('Timeout cannot be used when an existing ' - 'cluster is passed') - self._cluster = cluster.copy() - - if autoconnect: - self.open() - - self._initialized = True - - @staticmethod - def _handle_legacy_args(arguments_dict): - """Check legacy HappyBase arguments and warn if set. - - :type arguments_dict: dict - :param arguments_dict: Unused keyword arguments. - - :raises: :class:`TypeError ` if a keyword other - than ``host``, ``port``, ``compat``, ``transport`` or - ``protocol`` is used. - """ - common_args = _LEGACY_ARGS.intersection(six.iterkeys(arguments_dict)) - if common_args: - all_args = ', '.join(common_args) - message = ('The HappyBase legacy arguments %s were used. These ' - 'arguments are unused by gcloud.' % (all_args,)) - _WARN(message) - for arg_name in common_args: - arguments_dict.pop(arg_name) - if arguments_dict: - unexpected_names = arguments_dict.keys() - raise TypeError('Received unexpected arguments', unexpected_names) - - def open(self): - """Open the underlying transport to Cloud Bigtable. - - This method opens the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by - this connection. - """ - self._cluster._client.start() - - def close(self): - """Close the underlying transport to Cloud Bigtable. - - This method closes the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by - this connection. - """ - self._cluster._client.stop() - - def __del__(self): - if self._cluster is not None: - self.close() - - def _table_name(self, name): - """Construct a table name by optionally adding a table name prefix. - - :type name: str - :param name: The name to have a prefix added to it. - - :rtype: str - :returns: The prefixed name, if the current connection has a table - prefix set. - """ - if self.table_prefix is None: - return name - - return self.table_prefix + self.table_prefix_separator + name - - def table(self, name, use_prefix=True): - """Table factory. - - :type name: str - :param name: The name of the table to be created. - - :type use_prefix: bool - :param use_prefix: Whether to use the table prefix (if any). - - :rtype: `Table ` - :returns: Table instance owned by this connection. - """ - if use_prefix: - name = self._table_name(name) - return Table(name, self) - - def tables(self): - """Return a list of table names available to this connection. - - .. note:: - - This lists every table in the cluster owned by this connection, - **not** every table that a given user may have access to. - - .. note:: - - If ``table_prefix`` is set on this connection, only returns the - table names which match that prefix. - - :rtype: list - :returns: List of string table names. - """ - low_level_table_instances = self._cluster.list_tables() - table_names = [table_instance.table_id - for table_instance in low_level_table_instances] - - # Filter using prefix, and strip prefix from names - if self.table_prefix is not None: - prefix = self._table_name('') - offset = len(prefix) - table_names = [name[offset:] for name in table_names - if name.startswith(prefix)] - - return table_names - - def create_table(self, name, families): - """Create a table. - - .. warning:: - - The only column family options from HappyBase that are able to be - used with Cloud Bigtable are ``max_versions`` and ``time_to_live``. - - .. note:: - - This method is **not** atomic. The Cloud Bigtable API separates - the creation of a table from the creation of column families. Thus - this method needs to send 1 request for the table creation and 1 - request for each column family. If any of these fails, the method - will fail, but the progress made towards completion cannot be - rolled back. - - Values in ``families`` represent column family options. In HappyBase, - these are dictionaries, corresponding to the ``ColumnDescriptor`` - structure in the Thrift API. The accepted keys are: - - * ``max_versions`` (``int``) - * ``compression`` (``str``) - * ``in_memory`` (``bool``) - * ``bloom_filter_type`` (``str``) - * ``bloom_filter_vector_size`` (``int``) - * ``bloom_filter_nb_hashes`` (``int``) - * ``block_cache_enabled`` (``bool``) - * ``time_to_live`` (``int``) - - :type name: str - :param name: The name of the table to be created. - - :type families: dict - :param families: Dictionary with column family names as keys and column - family options as the values. The options can be among - - * :class:`dict` - * :class:`.GarbageCollectionRule` - - :raises: :class:`TypeError ` if ``families`` is - not a dictionary, - :class:`ValueError ` if ``families`` - has no entries - """ - if not isinstance(families, dict): - raise TypeError('families arg must be a dictionary') - - if not families: - raise ValueError('Cannot create table %r (no column ' - 'families specified)' % (name,)) - - # Parse all keys before making any API requests. - gc_rule_dict = {} - for column_family_name, option in families.items(): - if isinstance(column_family_name, six.binary_type): - column_family_name = column_family_name.decode('utf-8') - if column_family_name.endswith(':'): - column_family_name = column_family_name[:-1] - gc_rule_dict[column_family_name] = _parse_family_option(option) - - # Create table instance and then make API calls. - name = self._table_name(name) - low_level_table = _LowLevelTable(name, self._cluster) - low_level_table.create() - - for column_family_name, gc_rule in gc_rule_dict.items(): - column_family = low_level_table.column_family( - column_family_name, gc_rule=gc_rule) - column_family.create() - - def delete_table(self, name, disable=False): - """Delete the specified table. - - :type name: str - :param name: The name of the table to be deleted. If ``table_prefix`` - is set, a prefix will be added to the ``name``. - - :type disable: bool - :param disable: Whether to first disable the table if needed. This - is provided for compatibility with HappyBase, but is - not relevant for Cloud Bigtable since it has no concept - of enabled / disabled tables. - - :raises: :class:`ValueError ` - if ``disable=True``. - """ - if disable: - raise ValueError('The disable argument should not be used in ' - 'delete_table(). Cloud Bigtable has no concept ' - 'of enabled / disabled tables.') - - name = self._table_name(name) - _LowLevelTable(name, self._cluster).delete() - - def enable_table(self, name): - """Enable the specified table. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def disable_table(self, name): - """Disable the specified table. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def is_table_enabled(self, name): - """Return whether the specified table is enabled. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def compact_table(self, name, major=False): - """Compact the specified table. - - Cloud Bigtable does not support compacting a table, so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API does not support ' - 'compacting a table.') - - -def _parse_family_option(option): - """Parses a column family option into a garbage collection rule. - - .. note:: - - If ``option`` is not a dictionary, the type is not checked. - If ``option`` is :data:`None`, there is nothing to do, since this - is the correct output. - - :type option: :class:`dict`, - :data:`NoneType `, - :class:`.GarbageCollectionRule` - :param option: A column family option passes as a dictionary value in - :meth:`Connection.create_table`. - - :rtype: :class:`.GarbageCollectionRule` - :returns: A garbage collection rule parsed from the input. - """ - result = option - if isinstance(result, dict): - if not set(result.keys()) <= set(['max_versions', 'time_to_live']): - all_keys = ', '.join(repr(key) for key in result.keys()) - warning_msg = ('Cloud Bigtable only supports max_versions and ' - 'time_to_live column family settings. ' - 'Received: %s' % (all_keys,)) - _WARN(warning_msg) - - max_num_versions = result.get('max_versions') - max_age = None - if 'time_to_live' in result: - max_age = datetime.timedelta(seconds=result['time_to_live']) - - versions_rule = age_rule = None - if max_num_versions is not None: - versions_rule = MaxVersionsGCRule(max_num_versions) - if max_age is not None: - age_rule = MaxAgeGCRule(max_age) - - if versions_rule is None: - result = age_rule - else: - if age_rule is None: - result = versions_rule - else: - result = GCRuleIntersection(rules=[age_rule, versions_rule]) - - return result diff --git a/gcloud_bigtable/happybase/notes.txt b/gcloud_bigtable/happybase/notes.txt deleted file mode 100644 index bdc5af1..0000000 --- a/gcloud_bigtable/happybase/notes.txt +++ /dev/null @@ -1,98 +0,0 @@ -# Installing HBase / Thrift locally - - 1. Run `javac -version` to make sure you have JDK 7 (or 8) - For example "javac 1.7.0_79" == JDK 7. - 2. Visit http://apache.cs.utah.edu/hbase/ (or other mirror) - and find your latest version (e.g. - http://apache.cs.utah.edu/hbase/1.1.1/hbase-1.1.1-bin.tar.gz) - Make sure it is "-bin" and not "-src". - 3. Run `wget http://apache.cs.utah.edu/hbase/1.1.1/hbase-1.1.1-bin.tar.gz` - (HOLY SHIT I CANNOT BELIEVE NONE OF THE MIRRORS ARE HTTPS) - 4. Run `tar -zxvf hbase-1.1.1-bin.tar.gz` (maybe drop the -v flag) - 5. Set the JAVA_HOME environment variable (e.g. if `which java` tells - us it is in `/usr/bin/java` then `export JAVA_HOME=/usr` - 6. Edit `hbase-1.1.1/conf/hbase-site.xml` to give locations on the local - filesystem where HBase and ZooKeeper are allowed to write data - - - - hbase.rootdir - file://.../hbase_throwaway/hbase - - - hbase.zookeeper.property.dataDir - .../hbase_throwaway/zookeeper - - - - **DO NOT** create the HBase data directory. HBase will do this for you. - If you create the directory, HBase will attempt to do a migration, which - is not what you want. - - 7. Run `./hbase-1.1.1/bin/start-hbase.sh`. You can use the `jps` command to - verify that you have one running process called `HMaster`. (`jps` == - Java Process Status tool) - 8. Feel free to play around in the shell `./hbase-1.1.1/bin/hbase shell` - 9. Run `./hbase-1.1.1/bin/stop-hbase.sh` to step the server -10. Run `./hbase-1.1.1/bin/hbase thrift start` - (H/T: http://wiki.apache.org/hadoop/Hbase/ThriftApi) - -# Installing HappyBase locally - - 1. Run `virtualenv happybase-venv` - 2. Run `source happybase-venv/bin/activate` - 3. Run `pip install happybase` - 4. Run `python -c "import happybase"` to make sure it worked - 5. Run `pip install ipython` if you prefer that as your shell - in your virtualenv - -# Running HappyBase locally - - 1. Run `./hbase-1.1.1/bin/start-hbase.sh` - 2. Run `./hbase-1.1.1/bin/hbase thrift start --port=9090 > ./hbase-1.1.1/logs/thrift.log 2>&1 &` - (in the background, use `jps` to get the process ID) - 3. Run Python: - - import happybase - connection = happybase.Connection(host='localhost', port=9090) - print(connection.tables()) - families = {'family': {}} - table_name = 'table-name' - connection.create_table(table_name, families) - print(connection.tables()) - table = connection.table(table_name) - - row_key = 'row-key' - table.put(row_key, {'family:qual1': 'value1', - 'family:qual2': 'value2'}) - row = table.row(row_key) - print(type(row)) - - for key, data in table.scan(row_prefix='row'): - print((key, data)) - - table.put(row_key, {'family:qual1': 'value1-new', - 'family:qual2': 'value2-new'}) - row = table.row(row_key, include_timestamp=True) - print(row) - print(table.cells(row_key, 'family:qual1', include_timestamp=True)) - print(table.cells(row_key, 'family:qual2', include_timestamp=True)) - - print(list(table.scan())) - print(list(table.scan(sorted_columns=True))) - - - - -TESTING IF DELETE TIMESTAMPS MATTER -table.cells(row_key, 'family:qual1', include_timestamp=True) -table.put(row_key, {'family:qual1': 'new-data'}) -table.cells(row_key, 'family:qual1', include_timestamp=True) -table.delete(row_key, columns=['family:qual1']) -table.cells(row_key, 'family:qual1', include_timestamp=True) -table.put(row_key, {'family:qual1': 'value1'}) -table.put(row_key, {'family:qual1': 'new-data'}) -c1, c2 = table.cells(row_key, 'family:qual1', include_timestamp=True) -midpt = (c1[1] + c2[1]) // 2 -table.delete(row_key, columns=['family:qual1'], timestamp=midpt) -table.cells(row_key, 'family:qual1', include_timestamp=True) diff --git a/gcloud_bigtable/happybase/pool.py b/gcloud_bigtable/happybase/pool.py deleted file mode 100644 index e77d7ba..0000000 --- a/gcloud_bigtable/happybase/pool.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase pool module.""" - - -import contextlib -import threading - -import six - -from gcloud_bigtable.happybase.connection import Connection -from gcloud_bigtable.happybase.connection import _get_cluster - - -_MIN_POOL_SIZE = 1 -"""Minimum allowable size of a connection pool.""" - - -class NoConnectionsAvailable(RuntimeError): - """Exception raised when no connections are available. - - This happens if a timeout was specified when obtaining a connection, - and no connection became available within the specified timeout. - """ - - -class ConnectionPool(object): - """Thread-safe connection pool. - - .. note:: - - All keyword arguments are passed unmodified to the - :class:`.Connection` constructor **except** for ``autoconnect``. - This is because the ``open`` / ``closed`` status of a connection - is managed by the pool. In addition, if ``cluster`` is not passed, - the default / inferred cluster is determined by the pool and then - passed to each :class:`.Connection` that is created. - - :type size: int - :param size: The maximum number of concurrently open connections. - - :type kwargs: dict - :param kwargs: Keyword arguments passed to :class:`.Connection` - constructor. - - :raises: :class:`TypeError ` if ``size`` - is non an integer. - :class:`ValueError ` if ``size`` - is not positive. - """ - def __init__(self, size, **kwargs): - if not isinstance(size, six.integer_types): - raise TypeError('Pool size arg must be an integer') - - if size < _MIN_POOL_SIZE: - raise ValueError('Pool size must be positive') - - self._lock = threading.Lock() - self._queue = six.moves.queue.LifoQueue(maxsize=size) - self._thread_connections = threading.local() - - connection_kwargs = kwargs - connection_kwargs['autoconnect'] = False - if 'cluster' not in connection_kwargs: - connection_kwargs['cluster'] = _get_cluster( - timeout=kwargs.get('timeout')) - - for _ in six.moves.range(size): - connection = Connection(**connection_kwargs) - self._queue.put(connection) - - def _acquire_connection(self, timeout=None): - """Acquire a connection from the pool. - - :type timeout: int - :param timeout: (Optional) Time (in seconds) to wait for a connection - to open. - - :rtype: :class:`.Connection` - :returns: An active connection from the queue stored on the pool. - :raises: :class:`NoConnectionsAvailable` if ``Queue.get`` fails - before the ``timeout`` (only if a timeout is specified). - """ - try: - return self._queue.get(block=True, timeout=timeout) - except six.moves.queue.Empty: - raise NoConnectionsAvailable('No connection available from pool ' - 'within specified timeout') - - @contextlib.contextmanager - def connection(self, timeout=None): - """Obtain a connection from the pool. - - Must be used as a context manager, for example:: - - with pool.connection() as connection: - pass # do something with the connection - - If ``timeout`` is omitted, this method waits forever for a connection - to become available. - - :type timeout: int - :param timeout: (Optional) Time (in seconds) to wait for a connection - to open. - - :rtype: :class:`.Connection` - :returns: An active connection from the pool. - :raises: :class:`NoConnectionsAvailable` if no connection can be - retrieved from the pool before the ``timeout`` (only if - a timeout is specified). - """ - connection = getattr(self._thread_connections, 'current', None) - - retrieved_new_cnxn = False - if connection is None: - # In this case we need to actually grab a connection from the - # pool. After retrieval, the connection is stored on a thread - # local so that nested connection requests from the same - # thread can re-use the same connection instance. - # - # NOTE: This code acquires a lock before assigning to the - # thread local; see - # ('https://emptysqua.re/blog/' - # 'another-thing-about-pythons-threadlocals/') - retrieved_new_cnxn = True - connection = self._acquire_connection(timeout) - with self._lock: - self._thread_connections.current = connection - - # This is a no-op for connections that have already been opened - # since they just call Client.start(). - connection.open() - yield connection - - # Remove thread local reference after the outermost 'with' block - # ends. Afterwards the thread no longer owns the connection. - if retrieved_new_cnxn: - del self._thread_connections.current - self._queue.put(connection) diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py deleted file mode 100644 index 86046b0..0000000 --- a/gcloud_bigtable/happybase/table.py +++ /dev/null @@ -1,963 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase table module.""" - - -import struct -import warnings - -import six - -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds -from gcloud_bigtable._non_upstream_helpers import _to_bytes -from gcloud_bigtable._non_upstream_helpers import _total_seconds -from gcloud_bigtable.column_family import GCRuleIntersection -from gcloud_bigtable.column_family import MaxAgeGCRule -from gcloud_bigtable.column_family import MaxVersionsGCRule -from gcloud_bigtable.happybase.batch import Batch -from gcloud_bigtable.happybase.batch import _WAL_SENTINEL -from gcloud_bigtable.happybase.batch import _get_column_pairs -from gcloud_bigtable.row import CellsColumnLimitFilter -from gcloud_bigtable.row import ColumnQualifierRegexFilter -from gcloud_bigtable.row import FamilyNameRegexFilter -from gcloud_bigtable.row import RowFilterChain -from gcloud_bigtable.row import RowFilterUnion -from gcloud_bigtable.row import RowKeyRegexFilter -from gcloud_bigtable.row import TimestampRange -from gcloud_bigtable.row import TimestampRangeFilter -from gcloud_bigtable.table import Table as _LowLevelTable - - -_WARN = warnings.warn -_UNPACK_I64 = struct.Struct('>q').unpack -_SIMPLE_GC_RULES = (MaxAgeGCRule, MaxVersionsGCRule) -# Upstream renames -# pylint: disable=invalid-name -_datetime_from_microseconds = _microseconds_to_timestamp -_microseconds_from_datetime = _timestamp_to_microseconds -# pylint: enable=invalid-name - - -def make_row(cell_map, include_timestamp): - """Make a row dict for a Thrift cell mapping. - - .. note:: - - This method is only provided for HappyBase compatibility, but does not - actually work. - - :type cell_map: dict - :param cell_map: Dictionary with ``fam:col`` strings as keys and ``TCell`` - instances as values. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API output is not the same ' - 'as the output from the Thrift server, so this ' - 'helper can not be implemented.', 'Called with', - cell_map, include_timestamp) - - -def make_ordered_row(sorted_columns, include_timestamp): - """Make a row dict for sorted Thrift column results from scans. - - .. note:: - - This method is only provided for HappyBase compatibility, but does not - actually work. - - :type sorted_columns: list - :param sorted_columns: List of ``TColumn`` instances from Thrift. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API output is not the same ' - 'as the output from the Thrift server, so this ' - 'helper can not be implemented.', 'Called with', - sorted_columns, include_timestamp) - - -class Table(object): - """Representation of Cloud Bigtable table. - - Used for adding data and - - :type name: str - :param name: The name of the table. - - :type connection: :class:`.Connection` - :param connection: The connection which has access to the table. - """ - - def __init__(self, name, connection): - self.name = name - # This remains as legacy for HappyBase, but only the cluster - # from the connection is needed. - self.connection = connection - self._low_level_table = None - if self.connection is not None: - self._low_level_table = _LowLevelTable(self.name, - self.connection._cluster) - - def __repr__(self): - return '' % (self.name,) - - def families(self): - """Retrieve the column families for this table. - - :rtype: dict - :returns: Mapping from column family name to garbage collection rule - for a column family. - """ - column_family_map = self._low_level_table.list_column_families() - result = {} - for col_fam, col_fam_obj in six.iteritems(column_family_map): - result[col_fam] = _gc_rule_to_dict(col_fam_obj.gc_rule) - return result - - def regions(self): - """Retrieve the regions for this table. - - Cloud Bigtable does not give information about how a table is laid - out in memory, so regions so this method does not work. It is - provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API does not have a ' - 'concept of splitting a table into regions.') - - def row(self, row, columns=None, timestamp=None, include_timestamp=False): - """Retrieve a single row of data. - - Returns the latest cells in each column (or all columns if ``columns`` - is not specified). If a ``timestamp`` is set, then **latest** becomes - **latest** up until ``timestamp``. - - :type row: str - :param row: Row key for the row we are reading from. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before the - the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: dict - :returns: Dictionary containing all the latest column values in - the row. - """ - filters = [] - if columns is not None: - filters.append(_columns_filter_helper(columns)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_row_data = self._low_level_table.read_row( - row, filter_=filter_) - if partial_row_data is None: - return {} - - return _partial_row_to_dict(partial_row_data, - include_timestamp=include_timestamp) - - def rows(self, rows, columns=None, timestamp=None, - include_timestamp=False): - """Retrieve multiple rows of data. - - All optional arguments behave the same in this method as they do in - :meth:`row`. - - :type rows: list - :param rows: Iterable of the row keys for the rows we are reading from. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: A list of pairs, where the first is the row key and the - second is a dictionary with the filtered values returned. - """ - if not rows: - # Avoid round-trip if the result is empty anyway - return [] - - filters = [] - if columns is not None: - filters.append(_columns_filter_helper(columns)) - filters.append(_row_keys_filter_helper(rows)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_rows_data = self._low_level_table.read_rows(filter_=filter_) - # NOTE: We could use max_loops = 1000 or some similar value to ensure - # that the stream isn't open too long. - partial_rows_data.consume_all() - - result = [] - for row_key in rows: - if row_key not in partial_rows_data.rows: - continue - curr_row_data = partial_rows_data.rows[row_key] - curr_row_dict = _partial_row_to_dict( - curr_row_data, include_timestamp=include_timestamp) - result.append((row_key, curr_row_dict)) - - return result - - def cells(self, row, column, versions=None, timestamp=None, - include_timestamp=False): - """Retrieve multiple versions of a single cell from the table. - - :type row: str - :param row: Row key for the row we are reading from. - - :type column: str - :param column: Column we are reading from; of the form ``fam:col``. - - :type versions: int - :param versions: (Optional) The maximum number of cells to return. If - not set, returns all cells found. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: List of values in the cell (with timestamps if - ``include_timestamp`` is :data:`True`). - """ - filter_ = _filter_chain_helper(column=column, versions=versions, - timestamp=timestamp) - partial_row_data = self._low_level_table.read_row(row, filter_=filter_) - if partial_row_data is None: - return [] - else: - cells = partial_row_data._cells - # We know that `_filter_chain_helper` has already verified that - # column will split as such. - column_family_id, column_qualifier = column.split(':') - # NOTE: We expect the only key in `cells` is `column_family_id` - # and the only key `cells[column_family_id]` is - # `column_qualifier`. But we don't check that this is true. - curr_cells = cells[column_family_id][column_qualifier] - return _cells_to_pairs( - curr_cells, include_timestamp=include_timestamp) - - def scan(self, row_start=None, row_stop=None, row_prefix=None, - columns=None, filter=None, timestamp=None, - include_timestamp=False, limit=None, **kwargs): - """Create a scanner for data in this table. - - This method returns a generator that can be used for looping over the - matching rows. - - If ``row_prefix`` is specified, only rows with row keys matching the - prefix will be returned. If given, ``row_start`` and ``row_stop`` - cannot be used. - - .. note:: - - Both ``row_start`` and ``row_stop`` can be :data:`None` to specify - the start and the end of the table respectively. If both are - omitted, a full table scan is done. Note that this usually results - in severe performance problems. - - The arguments ``batch_size``, ``scan_batching`` and ``sorted_columns`` - are allowed (as keyword arguments) for compatibility with - HappyBase. However, they will not be used in any way, and will cause a - warning if passed. (The ``batch_size`` determines the number of - results to retrieve per request. The HBase scanner defaults to reading - one record at a time, so this argument allows HappyBase to increase - that number. However, the Cloud Bigtable API uses HTTP/2 streaming so - there is no concept of a batch. The ``sorted_columns`` flag tells - HBase to return columns in order, but Cloud Bigtable doesn't have - this feature.) - - :type row_start: str - :param row_start: (Optional) Row key where the scanner should start - (includes ``row_start``). If not specified, reads - from the first key. If the table does not contain - ``row_start``, it will start from the next key after - it that **is** contained in the table. - - :type row_stop: str - :param row_stop: (Optional) Row key where the scanner should stop - (excludes ``row_stop``). If not specified, reads - until the last key. The table does not have to contain - ``row_stop``. - - :type row_prefix: str - :param row_prefix: (Optional) Prefix to match row keys. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type filter: :class:`.RowFilter` - :param filter: (Optional) An additional filter (beyond column and - row range filters supported here). HappyBase / HBase - users will have used this as an HBase filter string. See - http://hbase.apache.org/0.94/book/thrift.html - for more details on those filters. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :type limit: int - :param limit: (Optional) Maximum number of rows to return. - - :type kwargs: dict - :param kwargs: Remaining keyword arguments. Provided for HappyBase - compatibility. - - :raises: If ``limit`` is set but non-positive, or if row prefix is - used with row start/stop, - :class:`TypeError ` if a string - ``filter`` is used. - """ - legacy_args = [] - for kw_name in ('batch_size', 'scan_batching', 'sorted_columns'): - if kw_name in kwargs: - legacy_args.append(kw_name) - kwargs.pop(kw_name) - if legacy_args: - legacy_args = ', '.join(legacy_args) - message = ('The HappyBase legacy arguments %s were used. These ' - 'arguments are unused by gcloud.' % (legacy_args,)) - _WARN(message) - if kwargs: - raise TypeError('Received unexpected arguments', kwargs.keys()) - - if limit is not None and limit < 1: - raise ValueError('limit must be positive') - if row_prefix is not None: - if row_start is not None or row_stop is not None: - raise ValueError('row_prefix cannot be combined with ' - 'row_start or row_stop') - row_start = row_prefix - row_stop = _string_successor(row_prefix) - - filters = [] - if isinstance(filter, six.string_types): - raise TypeError('HBase filter strings not supported by Cloud ' - 'Bigtable. RowFilter\'s from row module may be ' - 'used instead.') - elif filter is not None: - filters.append(filter) - - if columns is not None: - filters.append(_columns_filter_helper(columns)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_rows_data = self._low_level_table.read_rows( - start_key=row_start, end_key=row_stop, - limit=limit, filter_=filter_) - - # Mutable copy of data. - rows_dict = partial_rows_data.rows - while True: - try: - partial_rows_data.consume_next() - row_key, curr_row_data = rows_dict.popitem() - # NOTE: We expect len(rows_dict) == 0, but don't check it. - curr_row_dict = _partial_row_to_dict( - curr_row_data, include_timestamp=include_timestamp) - yield (row_key, curr_row_dict) - except StopIteration: - break - - def put(self, row, data, timestamp=None, wal=_WAL_SENTINEL): - """Insert data into a row in this table. - - .. note:: - - This method will send a request with a single "put" mutation. - In many situations, :meth:`batch` is a more appropriate - method to manipulate data since it helps combine many mutations - into a single request. - - :type row: str - :param row: The row key where the mutation will be "put". - - :type data: dict - :param data: Dictionary containing the data to be inserted. The keys - are columns names (of the form ``fam:col``) and the values - are strings (bytes) to be stored in those columns. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that the mutation will be applied at. - - :type wal: object - :param wal: Unused parameter (to be passed to a created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - """ - with self.batch(timestamp=timestamp, wal=wal) as batch: - batch.put(row, data) - - def delete(self, row, columns=None, timestamp=None, wal=_WAL_SENTINEL): - """Delete data from a row in this table. - - This method deletes the entire ``row`` if ``columns`` is not - specified. - - .. note:: - - This method will send a request with a single delete mutation. - In many situations, :meth:`batch` is a more appropriate - method to manipulate data since it helps combine many mutations - into a single request. - - :type row: str - :param row: The row key where the delete will occur. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that the mutation will be applied at. - - :type wal: object - :param wal: Unused parameter (to be passed to a created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - """ - with self.batch(timestamp=timestamp, wal=wal) as batch: - batch.delete(row, columns) - - def batch(self, timestamp=None, batch_size=None, transaction=False, - wal=_WAL_SENTINEL): - """Create a new batch operation for this table. - - This method returns a new :class:`.Batch` instance that can be used - for mass data manipulation. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that all mutations will be applied at. - - :type batch_size: int - :param batch_size: (Optional) The maximum number of mutations to allow - to accumulate before committing them. - - :type transaction: bool - :param transaction: Flag indicating if the mutations should be sent - transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. - - :type wal: object - :param wal: Unused parameter (to be passed to the created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - - :rtype: :class:`gcloud.bigtable.happybase.batch.Batch` - :returns: A batch bound to this table. - """ - return Batch(self, timestamp=timestamp, batch_size=batch_size, - transaction=transaction, wal=wal) - - def counter_get(self, row, column): - """Retrieve the current value of a counter column. - - This method retrieves the current value of a counter column. If the - counter column does not exist, this function initializes it to ``0``. - - .. note:: - - Application code should **never** store a counter value directly; - use the atomic :meth:`counter_inc` and :meth:`counter_dec` methods - for that. - - :type row: str - :param row: Row key for the row we are getting a counter from. - - :type column: str - :param column: Column we are ``get``-ing from; of the form ``fam:col``. - - :rtype: int - :returns: Counter value (after initializing / incrementing by 0). - """ - # Don't query directly, but increment with value=0 so that the counter - # is correctly initialized if didn't exist yet. - return self.counter_inc(row, column, value=0) - - def counter_set(self, row, column, value=0): - """Set a counter column to a specific value. - - This method is provided in HappyBase, but we do not provide it here - because it defeats the purpose of using atomic increment and decrement - of a counter. - - :type row: str - :param row: Row key for the row we are setting a counter in. - - :type column: str - :param column: Column we are setting a value in; of - the form ``fam:col``. - - :type value: int - :param value: Value to set the counter to. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('Table.counter_set will not be implemented. ' - 'Instead use the increment/decrement ' - 'methods along with counter_get.') - - def counter_inc(self, row, column, value=1): - """Atomically increment a counter column. - - This method atomically increments a counter column in ``row``. - If the counter column does not exist, it is automatically initialized - to ``0`` before being incremented. - - :type row: str - :param row: Row key for the row we are incrementing a counter in. - - :type column: str - :param column: Column we are incrementing a value in; of the - form ``fam:col``. - - :type value: int - :param value: Amount to increment the counter by. (If negative, - this is equivalent to decrement.) - - :rtype: int - :returns: Counter value after incrementing. - """ - row = self._low_level_table.row(row) - if isinstance(column, six.binary_type): - column = column.decode('utf-8') - column_family_id, column_qualifier = column.split(':') - row.increment_cell_value(column_family_id, column_qualifier, value) - # See row.commit_modifications() will return a dictionary: - # { - # u'col-fam-id': { - # b'col-name1': [ - # (b'cell-val', datetime.datetime(...)), - # ... - # ], - # ... - # }, - # } - modified_cells = row.commit_modifications() - # Get the cells in the modified column, - column_cells = modified_cells[column_family_id][column_qualifier] - # Make sure there is exactly one cell in the column. - if len(column_cells) != 1: - raise ValueError('Expected server to return one modified cell.') - column_cell = column_cells[0] - # Get the bytes value from the column and convert it to an integer. - bytes_value = column_cell[0] - int_value, = _UNPACK_I64(bytes_value) - return int_value - - def counter_dec(self, row, column, value=1): - """Atomically decrement a counter column. - - This method atomically decrements a counter column in ``row``. - If the counter column does not exist, it is automatically initialized - to ``0`` before being decremented. - - :type row: str - :param row: Row key for the row we are decrementing a counter in. - - :type column: str - :param column: Column we are decrementing a value in; of the - form ``fam:col``. - - :type value: int - :param value: Amount to decrement the counter by. (If negative, - this is equivalent to increment.) - - :rtype: int - :returns: Counter value after decrementing. - """ - return self.counter_inc(row, column, -value) - - -def _gc_rule_to_dict(gc_rule): - """Converts garbage collection rule to dictionary if possible. - - This is in place to support dictionary values as was done - in HappyBase, which has somewhat different garbage collection rule - settings for column families. - - Only does this if the garbage collection rule is: - - * :class:`.MaxAgeGCRule` - * :class:`.MaxVersionsGCRule` - * Composite :class:`.GCRuleIntersection` with two rules, one each - of type :class:`.MaxAgeGCRule` and :class:`.MaxVersionsGCRule` - - Otherwise, just returns the input without change. - - :type gc_rule: :data:`NoneType `, - :class:`.GarbageCollectionRule` - :param gc_rule: A garbage collection rule to convert to a dictionary - (if possible). - - :rtype: dict or :class:`.GarbageCollectionRule` - :returns: The converted garbage collection rule. - """ - result = gc_rule - if gc_rule is None: - result = {} - elif isinstance(gc_rule, MaxAgeGCRule): - result = {'time_to_live': _total_seconds(gc_rule.max_age)} - elif isinstance(gc_rule, MaxVersionsGCRule): - result = {'max_versions': gc_rule.max_num_versions} - elif isinstance(gc_rule, GCRuleIntersection): - if len(gc_rule.rules) == 2: - rule1, rule2 = gc_rule.rules - if (isinstance(rule1, _SIMPLE_GC_RULES) and - isinstance(rule2, _SIMPLE_GC_RULES)): - rule1 = _gc_rule_to_dict(rule1) - rule2 = _gc_rule_to_dict(rule2) - key1, = rule1.keys() - key2, = rule2.keys() - if key1 != key2: - result = {key1: rule1[key1], key2: rule2[key2]} - return result - - -def _next_char(str_val, index): - """Gets the next character based on a position in a string. - - :type str_val: str - :param str_val: A string containing the character to update. - - :type index: int - :param index: An integer index in ``str_val``. - - :rtype: str - :returns: The next character after the character at ``index`` - in ``str_val``. - """ - ord_val = six.indexbytes(str_val, index) - return _to_bytes(chr(ord_val + 1), encoding='latin-1') - - -def _string_successor(str_val): - """Increment and truncate a byte string. - - Determines shortest string that sorts after the given string when - compared using regular string comparison semantics. - - Modeled after implementation in ``gcloud-golang``. - - Increments the last byte that is smaller than ``0xFF``, and - drops everything after it. If the string only contains ``0xFF`` bytes, - ``''`` is returned. - - :type str_val: str - :param str_val: String to increment. - - :rtype: str - :returns: The next string in lexical order after ``str_val``. - """ - str_val = _to_bytes(str_val, encoding='latin-1') - if str_val == b'': - return str_val - - index = len(str_val) - 1 - while index >= 0: - if six.indexbytes(str_val, index) != 0xff: - break - index -= 1 - - if index == -1: - return b'' - - return str_val[:index] + _next_char(str_val, index) - - -def _convert_to_time_range(timestamp=None): - """Create a timestamp range from an HBase / HappyBase timestamp. - - HBase uses timestamp as an argument to specify an exclusive end - deadline. Cloud Bigtable also uses exclusive end times, so - the behavior matches. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). Intended to be used as the end of an HBase - time range, which is exclusive. - - :rtype: :class:`.TimestampRange`, :data:`NoneType ` - :returns: The timestamp range corresponding to the passed in - ``timestamp``. - """ - if timestamp is None: - return None - - next_timestamp = _datetime_from_microseconds(1000 * timestamp) - return TimestampRange(end=next_timestamp) - - -def _cells_to_pairs(cells, include_timestamp=False): - """Converts list of cells to HappyBase format. - - For example:: - - >>> import datetime - >>> from gcloud_bigtable.row_data import Cell - >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) - >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) - >>> _cells_to_pairs([cell1, cell2]) - [b'val1', b'val2'] - >>> _cells_to_pairs([cell1, cell2], include_timestamp=True) - [(b'val1', 1456361486255), (b'val2', 1456361491927)] - - :type cells: list - :param cells: List of :class:`.Cell` returned from a read request. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: List of values in the cell. If ``include_timestamp=True``, each - value will be a pair, with the first part the bytes value in - the cell and the second part the number of milliseconds in the - timestamp on the cell. - """ - result = [] - for cell in cells: - if include_timestamp: - ts_millis = _microseconds_from_datetime(cell.timestamp) // 1000 - result.append((cell.value, ts_millis)) - else: - result.append(cell.value) - return result - - -def _partial_row_to_dict(partial_row_data, include_timestamp=False): - """Convert a low-level row data object to a dictionary. - - Assumes only the latest value in each row is needed. This assumption - is due to the fact that this method is used by callers which use - a ``CellsColumnLimitFilter(1)`` filter. - - For example:: - - >>> import datetime - >>> from gcloud_bigtable.row_data import Cell, PartialRowData - >>> cell1 = Cell(b'val1', datetime.datetime.utcnow()) - >>> cell2 = Cell(b'val2', datetime.datetime.utcnow()) - >>> row_data = PartialRowData(b'row-key') - >>> _partial_row_to_dict(row_data) - {} - >>> row_data._cells[u'fam1'] = {b'col1': [cell1], b'col2': [cell2]} - >>> _partial_row_to_dict(row_data) - {b'fam1:col2': b'val2', b'fam1:col1': b'val1'} - >>> _partial_row_to_dict(row_data, include_timestamp=True) - {b'fam1:col2': (b'val2', 1456361724480), - b'fam1:col1': (b'val1', 1456361721135)} - - :type partial_row_data: :class:`.row_data.PartialRowData` - :param partial_row_data: Row data consumed from a stream. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: dict - :returns: The row data converted to a dictionary. - """ - result = {} - for column, cells in six.iteritems(partial_row_data.to_dict()): - cell_vals = _cells_to_pairs(cells, - include_timestamp=include_timestamp) - # NOTE: We assume there is exactly 1 version since we used that in - # our filter, but we don't check this. - result[column] = cell_vals[0] - return result - - -def _filter_chain_helper(column=None, versions=None, timestamp=None, - filters=None): - """Create filter chain to limit a results set. - - :type column: str - :param column: (Optional) The column (``fam:col``) to be selected - with the filter. - - :type versions: int - :param versions: (Optional) The maximum number of cells to return. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be matched. - - :type filters: list - :param filters: (Optional) List of existing filters to be extended. - - :rtype: :class:`.RowFilter` - :returns: The chained filter created, or just a single filter if only - one was needed. - :raises: :class:`ValueError ` if there are no - filters to chain. - """ - if filters is None: - filters = [] - - if column is not None: - if isinstance(column, six.binary_type): # pragma: NO COVER - column = column.decode('utf-8') - column_family_id, column_qualifier = column.split(':') - fam_filter = FamilyNameRegexFilter(column_family_id) - qual_filter = ColumnQualifierRegexFilter(column_qualifier) - filters.extend([fam_filter, qual_filter]) - if versions is not None: - filters.append(CellsColumnLimitFilter(versions)) - time_range = _convert_to_time_range(timestamp=timestamp) - if time_range is not None: - filters.append(TimestampRangeFilter(time_range)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterChain(filters=filters) - - -def _columns_filter_helper(columns): - """Creates a union filter for a list of columns. - - :type columns: list - :param columns: Iterable containing column names (as strings). Each column - name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :rtype: :class:`.RowFilter` - :returns: The union filter created containing all of the matched columns. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for column_family_id, column_qualifier in _get_column_pairs(columns): - fam_filter = FamilyNameRegexFilter(column_family_id) - if column_qualifier is not None: - qual_filter = ColumnQualifierRegexFilter(column_qualifier) - combined_filter = RowFilterChain( - filters=[fam_filter, qual_filter]) - filters.append(combined_filter) - else: - filters.append(fam_filter) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) - - -def _row_keys_filter_helper(row_keys): - """Creates a union filter for a list of rows. - - :type row_keys: list - :param row_keys: Iterable containing row keys (as strings). - - :rtype: :class:`.RowFilter` - :returns: The union filter created containing all of the row keys. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for row_key in row_keys: - filters.append(RowKeyRegexFilter(row_key)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) diff --git a/gcloud_bigtable/happybase/test_batch.py b/gcloud_bigtable/happybase/test_batch.py deleted file mode 100644 index 69f81a4..0000000 --- a/gcloud_bigtable/happybase/test_batch.py +++ /dev/null @@ -1,559 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class _SendMixin(object): - - _send_called = False - - def send(self): - self._send_called = True - - -class Test__get_column_pairs(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.batch import _get_column_pairs - return _get_column_pairs(*args, **kwargs) - - def test_it(self): - columns = ['cf1', 'cf2:', 'cf3::', 'cf3:name1', 'cf3:name2'] - result = self._callFUT(columns) - expected_result = [ - ['cf1', None], - ['cf2', None], - ['cf3', ''], - ['cf3', 'name1'], - ['cf3', 'name2'], - ] - self.assertEqual(result, expected_result) - - def test_bad_column(self): - columns = ['a:b:c'] - with self.assertRaises(ValueError): - self._callFUT(columns) - - def test_bad_column_type(self): - columns = [None] - with self.assertRaises(AttributeError): - self._callFUT(columns) - - def test_bad_columns_var(self): - columns = None - with self.assertRaises(TypeError): - self._callFUT(columns) - - def test_column_family_with_require_qualifier(self): - columns = ['a:'] - with self.assertRaises(ValueError): - self._callFUT(columns, require_qualifier=True) - - -class TestBatch(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.batch import Batch - return Batch - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - table = object() - batch = self._makeOne(table) - self.assertEqual(batch._table, table) - self.assertEqual(batch._batch_size, None) - self.assertEqual(batch._timestamp, None) - self.assertEqual(batch._delete_range, None) - self.assertEqual(batch._transaction, False) - - def test_constructor_explicit(self): - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - from gcloud_bigtable.row import TimestampRange - - table = object() - timestamp = 144185290431 - batch_size = 42 - transaction = False # Must be False when batch_size is non-null - - batch = self._makeOne(table, timestamp=timestamp, - batch_size=batch_size, transaction=transaction) - self.assertEqual(batch._table, table) - self.assertEqual(batch._batch_size, batch_size) - self.assertEqual(batch._timestamp, - _microseconds_to_timestamp(1000 * timestamp)) - - next_timestamp = _microseconds_to_timestamp(1000 * (timestamp + 1)) - time_range = TimestampRange(end=next_timestamp) - self.assertEqual(batch._delete_range, time_range) - self.assertEqual(batch._transaction, transaction) - - def test_constructor_with_non_default_wal(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import batch as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - - table = object() - wal = object() - with _Monkey(MUT, _WARN=mock_warn): - self._makeOne(table, wal=wal) - - self.assertEqual(warned, [MUT._WAL_WARNING]) - - def test_constructor_with_non_positive_batch_size(self): - table = object() - batch_size = -10 - with self.assertRaises(ValueError): - self._makeOne(table, batch_size=batch_size) - batch_size = 0 - with self.assertRaises(ValueError): - self._makeOne(table, batch_size=batch_size) - - def test_constructor_with_batch_size_and_transactional(self): - table = object() - batch_size = 1 - transaction = True - with self.assertRaises(TypeError): - self._makeOne(table, batch_size=batch_size, - transaction=transaction) - - def test_send(self): - table = object() - batch = self._makeOne(table) - - batch._row_map = row_map = _MockRowMap() - row_map['row-key1'] = row1 = _MockRow() - row_map['row-key2'] = row2 = _MockRow() - batch._mutation_count = 1337 - - self.assertEqual(row_map.clear_count, 0) - self.assertEqual(row1.commits, 0) - self.assertEqual(row2.commits, 0) - self.assertNotEqual(batch._mutation_count, 0) - self.assertNotEqual(row_map, {}) - - batch.send() - self.assertEqual(row_map.clear_count, 1) - self.assertEqual(row1.commits, 1) - self.assertEqual(row2.commits, 1) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(row_map, {}) - - def test__try_send_no_batch_size(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - - self.assertEqual(batch._batch_size, None) - self.assertFalse(batch._send_called) - batch._try_send() - self.assertFalse(batch._send_called) - - def test__try_send_too_few_mutations(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch_size = 10 - batch = BatchWithSend(table, batch_size=batch_size) - - self.assertEqual(batch._batch_size, batch_size) - self.assertFalse(batch._send_called) - mutation_count = 2 - batch._mutation_count = mutation_count - self.assertTrue(mutation_count < batch_size) - batch._try_send() - self.assertFalse(batch._send_called) - - def test__try_send_actual_send(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch_size = 10 - batch = BatchWithSend(table, batch_size=batch_size) - - self.assertEqual(batch._batch_size, batch_size) - self.assertFalse(batch._send_called) - mutation_count = 12 - batch._mutation_count = mutation_count - self.assertTrue(mutation_count > batch_size) - batch._try_send() - self.assertTrue(batch._send_called) - - def test__get_row_exists(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - row_obj = object() - batch._row_map[row_key] = row_obj - result = batch._get_row(row_key) - self.assertEqual(result, row_obj) - - def test__get_row_create_new(self): - # Make mock batch and make sure we can create a low-level table. - low_level_table = _MockLowLevelTable() - table = _MockTable(low_level_table) - batch = self._makeOne(table) - - # Make sure row map is empty. - self.assertEqual(batch._row_map, {}) - - # Customize/capture mock table creation. - low_level_table.mock_row = mock_row = object() - - # Actually get the row (which creates a row via a low-level table). - row_key = 'row-key' - result = batch._get_row(row_key) - self.assertEqual(result, mock_row) - - # Check all the things that were constructed. - self.assertEqual(low_level_table.rows_made, [row_key]) - # Check how the batch was updated. - self.assertEqual(batch._row_map, {row_key: mock_row}) - - def test_put_bad_wal(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import batch as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - raise RuntimeError('No need to continue') - - table = object() - batch = self._makeOne(table) - - row = 'row-key' - data = {} - wal = None - - self.assertNotEqual(wal, MUT._WAL_SENTINEL) - with _Monkey(MUT, _WARN=mock_warn): - with self.assertRaises(RuntimeError): - batch.put(row, data, wal=wal) - self.assertEqual(warned, [MUT._WAL_WARNING]) - - def test_put(self): - import operator - - table = object() - batch = self._makeOne(table) - batch._timestamp = timestamp = object() - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - col1_fam = 'cf1' - col1_qual = 'qual1' - value1 = 'value1' - col2_fam = 'cf2' - col2_qual = 'qual2' - value2 = 'value2' - data = {col1_fam + ':' + col1_qual: value1, - col2_fam + ':' + col2_qual: value2} - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(row.set_cell_calls, []) - batch.put(row_key, data) - self.assertEqual(batch._mutation_count, 2) - # Since the calls depend on data.keys(), the order - # is non-deterministic. - first_elt = operator.itemgetter(0) - ordered_calls = sorted(row.set_cell_calls, key=first_elt) - - cell1_args = (col1_fam, col1_qual, value1) - cell1_kwargs = {'timestamp': timestamp} - cell2_args = (col2_fam, col2_qual, value2) - cell2_kwargs = {'timestamp': timestamp} - self.assertEqual(ordered_calls, [ - (cell1_args, cell1_kwargs), - (cell2_args, cell2_kwargs), - ]) - - def test_put_call_try_send(self): - klass = self._getTargetClass() - - class CallTrySend(klass): - - try_send_calls = 0 - - def _try_send(self): - self.try_send_calls += 1 - - table = object() - batch = CallTrySend(table) - - row_key = 'row-key' - batch._row_map[row_key] = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 0) - # No data so that nothing happens - batch.put(row_key, data={}) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 1) - - def _delete_columns_test_helper(self, time_range=None): - table = object() - batch = self._makeOne(table) - batch._delete_range = time_range - - col1_fam = 'cf1' - col2_fam = 'cf2' - col2_qual = 'col-name' - columns = [col1_fam + ':', col2_fam + ':' + col2_qual] - row_object = _MockRow() - - batch._delete_columns(columns, row_object) - self.assertEqual(row_object.commits, 0) - - cell_deleted_args = (col2_fam, col2_qual) - cell_deleted_kwargs = {'time_range': time_range} - self.assertEqual(row_object.delete_cell_calls, - [(cell_deleted_args, cell_deleted_kwargs)]) - fam_deleted_args = (col1_fam,) - fam_deleted_kwargs = {'columns': row_object.ALL_COLUMNS} - self.assertEqual(row_object.delete_cells_calls, - [(fam_deleted_args, fam_deleted_kwargs)]) - - def test__delete_columns(self): - self._delete_columns_test_helper() - - def test__delete_columns_w_time_and_col_fam(self): - time_range = object() - with self.assertRaises(ValueError): - self._delete_columns_test_helper(time_range=time_range) - - def test_delete_bad_wal(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import batch as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - raise RuntimeError('No need to continue') - - table = object() - batch = self._makeOne(table) - - row = 'row-key' - columns = [] - wal = None - - self.assertNotEqual(wal, MUT._WAL_SENTINEL) - with _Monkey(MUT, _WARN=mock_warn): - with self.assertRaises(RuntimeError): - batch.delete(row, columns=columns, wal=wal) - self.assertEqual(warned, [MUT._WAL_WARNING]) - - def test_delete_entire_row(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - batch.delete(row_key, columns=None) - self.assertEqual(row.deletes, 1) - self.assertEqual(batch._mutation_count, 1) - - def test_delete_entire_row_with_ts(self): - table = object() - batch = self._makeOne(table) - batch._delete_range = object() - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - with self.assertRaises(ValueError): - batch.delete(row_key, columns=None) - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - - def test_delete_call_try_send(self): - klass = self._getTargetClass() - - class CallTrySend(klass): - - try_send_calls = 0 - - def _try_send(self): - self.try_send_calls += 1 - - table = object() - batch = CallTrySend(table) - - row_key = 'row-key' - batch._row_map[row_key] = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 0) - # No columns so that nothing happens - batch.delete(row_key, columns=[]) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 1) - - def test_delete_some_columns(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - - col1_fam = 'cf1' - col2_fam = 'cf2' - col2_qual = 'col-name' - columns = [col1_fam + ':', col2_fam + ':' + col2_qual] - batch.delete(row_key, columns=columns) - - self.assertEqual(batch._mutation_count, 2) - cell_deleted_args = (col2_fam, col2_qual) - cell_deleted_kwargs = {'time_range': None} - self.assertEqual(row.delete_cell_calls, - [(cell_deleted_args, cell_deleted_kwargs)]) - fam_deleted_args = (col1_fam,) - fam_deleted_kwargs = {'columns': row.ALL_COLUMNS} - self.assertEqual(row.delete_cells_calls, - [(fam_deleted_args, fam_deleted_kwargs)]) - - def test_context_manager(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - self.assertFalse(batch._send_called) - - with batch: - pass - - self.assertTrue(batch._send_called) - - def test_context_manager_with_exception_non_transactional(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - self.assertFalse(batch._send_called) - - with self.assertRaises(ValueError): - with batch: - raise ValueError('Something bad happened') - - self.assertTrue(batch._send_called) - - def test_context_manager_with_exception_transactional(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table, transaction=True) - self.assertFalse(batch._send_called) - - with self.assertRaises(ValueError): - with batch: - raise ValueError('Something bad happened') - - self.assertFalse(batch._send_called) - - # Just to make sure send() actually works (and to make cover happy). - batch.send() - self.assertTrue(batch._send_called) - - -class _MockRowMap(dict): - - clear_count = 0 - - def clear(self): - self.clear_count += 1 - super(_MockRowMap, self).clear() - - -class _MockRow(object): - - ALL_COLUMNS = object() - - def __init__(self): - self.commits = 0 - self.deletes = 0 - self.set_cell_calls = [] - self.delete_cell_calls = [] - self.delete_cells_calls = [] - - def commit(self): - self.commits += 1 - - def delete(self): - self.deletes += 1 - - def set_cell(self, *args, **kwargs): - self.set_cell_calls.append((args, kwargs)) - - def delete_cell(self, *args, **kwargs): - self.delete_cell_calls.append((args, kwargs)) - - def delete_cells(self, *args, **kwargs): - self.delete_cells_calls.append((args, kwargs)) - - -class _MockTable(object): - - def __init__(self, low_level_table): - self._low_level_table = low_level_table - - -class _MockLowLevelTable(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.rows_made = [] - self.mock_row = None - - def row(self, row_key): - self.rows_made.append(row_key) - return self.mock_row diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py deleted file mode 100644 index 9aa2c13..0000000 --- a/gcloud_bigtable/happybase/test_connection.py +++ /dev/null @@ -1,608 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class Test__get_cluster(unittest2.TestCase): - - def _callFUT(self, timeout=None): - from gcloud_bigtable.happybase.connection import _get_cluster - return _get_cluster(timeout=timeout) - - def _helper(self, timeout=None, clusters=(), failed_zones=()): - from functools import partial - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - client_with_clusters = partial(_Client, clusters=clusters, - failed_zones=failed_zones) - with _Monkey(MUT, Client=client_with_clusters): - result = self._callFUT(timeout=timeout) - - # If we've reached this point, then _callFUT didn't fail, so we know - # there is exactly one cluster. - cluster, = clusters - self.assertEqual(result, cluster) - client = cluster.client - self.assertEqual(client.args, ()) - expected_kwargs = {'admin': True} - if timeout is not None: - expected_kwargs['timeout_seconds'] = timeout / 1000.0 - self.assertEqual(client.kwargs, expected_kwargs) - self.assertEqual(client.start_calls, 1) - self.assertEqual(client.stop_calls, 1) - - def test_default(self): - cluster = _Cluster() - self._helper(clusters=[cluster]) - - def test_with_timeout(self): - cluster = _Cluster() - self._helper(timeout=2103, clusters=[cluster]) - - def test_with_no_clusters(self): - with self.assertRaises(ValueError): - self._helper() - - def test_with_too_many_clusters(self): - clusters = [_Cluster(), _Cluster()] - with self.assertRaises(ValueError): - self._helper(clusters=clusters) - - def test_with_failed_zones(self): - cluster = _Cluster() - failed_zone = 'us-central1-c' - with self.assertRaises(ValueError): - self._helper(clusters=[cluster], - failed_zones=[failed_zone]) - - -class Test__parse_family_option(unittest2.TestCase): - - def _callFUT(self, option): - from gcloud_bigtable.happybase.connection import _parse_family_option - return _parse_family_option(option) - - def test_dictionary_no_keys(self): - option = {} - result = self._callFUT(option) - self.assertEqual(result, None) - - def test_null(self): - option = None - result = self._callFUT(option) - self.assertEqual(result, None) - - def test_dictionary_bad_key(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - - option = {'badkey': None} - with _Monkey(MUT, _WARN=mock_warn): - result = self._callFUT(option) - - self.assertEqual(result, None) - self.assertEqual(len(warned), 1) - self.assertIn('badkey', warned[0]) - - def test_dictionary_versions_key(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - - versions = 42 - option = {'max_versions': versions} - result = self._callFUT(option) - - gc_rule = MaxVersionsGCRule(versions) - self.assertEqual(result, gc_rule) - - def test_dictionary_ttl_key(self): - import datetime - from gcloud_bigtable.column_family import MaxAgeGCRule - - time_to_live = 24 * 60 * 60 - max_age = datetime.timedelta(days=1) - option = {'time_to_live': time_to_live} - result = self._callFUT(option) - - gc_rule = MaxAgeGCRule(max_age) - self.assertEqual(result, gc_rule) - - def test_dictionary_both_keys(self): - import datetime - from gcloud_bigtable.column_family import GCRuleIntersection - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - versions = 42 - time_to_live = 24 * 60 * 60 - option = { - 'max_versions': versions, - 'time_to_live': time_to_live, - } - result = self._callFUT(option) - - max_age = datetime.timedelta(days=1) - # NOTE: This relies on the order of the rules in the method we are - # calling matching this order here. - gc_rule1 = MaxAgeGCRule(max_age) - gc_rule2 = MaxVersionsGCRule(versions) - gc_rule = GCRuleIntersection(rules=[gc_rule1, gc_rule2]) - self.assertEqual(result, gc_rule) - - def test_non_dictionary(self): - option = object() - self.assertFalse(isinstance(option, dict)) - result = self._callFUT(option) - self.assertEqual(result, option) - - -class TestConnection(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.connection import Connection - return Connection - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - cluster = _Cluster() # Avoid implicit environ check. - self.assertEqual(cluster._client.start_calls, 0) - connection = self._makeOne(cluster=cluster) - self.assertEqual(cluster._client.start_calls, 1) - self.assertEqual(cluster._client.stop_calls, 0) - - self.assertEqual(connection._cluster, cluster) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - - def test_constructor_no_autoconnect(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster._client.start_calls, 0) - self.assertEqual(cluster._client.stop_calls, 0) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - - def test_constructor_missing_cluster(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() - timeout = object() - mock_get_cluster = _MockCalled(cluster) - with _Monkey(MUT, _get_cluster=mock_get_cluster): - connection = self._makeOne(autoconnect=False, cluster=None, - timeout=timeout) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - self.assertEqual(connection._cluster, cluster) - - mock_get_cluster.check_called(self, [()], [{'timeout': timeout}]) - - def test_constructor_explicit(self): - autoconnect = False - table_prefix = 'table-prefix' - table_prefix_separator = 'sep' - cluster_copy = _Cluster() - cluster = _Cluster(copies=[cluster_copy]) - - connection = self._makeOne( - autoconnect=autoconnect, - table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - self.assertEqual(connection.table_prefix, table_prefix) - self.assertEqual(connection.table_prefix_separator, - table_prefix_separator) - self.assertEqual(connection._cluster, cluster_copy) - - def test_constructor_with_unknown_argument(self): - cluster = _Cluster() - with self.assertRaises(TypeError): - self._makeOne(cluster=cluster, unknown='foo') - - def test_constructor_with_legacy_args(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - mock_warn = _MockCalled() - cluster = _Cluster() - with _Monkey(MUT, _WARN=mock_warn): - self._makeOne(cluster=cluster, host=object(), - port=object(), compat=object(), - transport=object(), protocol=object()) - - self.assertEqual(mock_warn.called_kwargs, [{}]) - self.assertEqual(len(mock_warn.called_args), 1) - self.assertEqual(len(mock_warn.called_args[0]), 1) - warning_msg = mock_warn.called_args[0][0] - self.assertIn('host', warning_msg) - self.assertIn('port', warning_msg) - self.assertIn('compat', warning_msg) - self.assertIn('transport', warning_msg) - self.assertIn('protocol', warning_msg) - - def test_constructor_with_timeout_and_cluster(self): - cluster = _Cluster() - with self.assertRaises(ValueError): - self._makeOne(cluster=cluster, timeout=object()) - - def test_constructor_non_string_prefix(self): - table_prefix = object() - - with self.assertRaises(TypeError): - self._makeOne(autoconnect=False, - table_prefix=table_prefix) - - def test_constructor_non_string_prefix_separator(self): - table_prefix_separator = object() - - with self.assertRaises(TypeError): - self._makeOne(autoconnect=False, - table_prefix_separator=table_prefix_separator) - - def test_open(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster._client.start_calls, 0) - connection.open() - self.assertEqual(cluster._client.start_calls, 1) - self.assertEqual(cluster._client.stop_calls, 0) - - def test_close(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster._client.stop_calls, 0) - connection.close() - self.assertEqual(cluster._client.stop_calls, 1) - self.assertEqual(cluster._client.start_calls, 0) - - def test___del__good_initialization(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster._client.stop_calls, 0) - connection.__del__() - self.assertEqual(cluster._client.stop_calls, 1) - - def test___del__bad_initialization(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - # Fake that initialization failed. - del connection._cluster - - self.assertEqual(cluster._client.stop_calls, 0) - connection.__del__() - self.assertEqual(cluster._client.stop_calls, 0) - - def test__table_name_with_prefix_set(self): - table_prefix = 'table-prefix' - table_prefix_separator = '<>' - cluster = _Cluster() - - connection = self._makeOne( - autoconnect=False, - table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - name = 'some-name' - prefixed = connection._table_name(name) - self.assertEqual(prefixed, - table_prefix + table_prefix_separator + name) - - def test__table_name_with_no_prefix_set(self): - cluster = _Cluster() - connection = self._makeOne(autoconnect=False, - cluster=cluster) - - name = 'some-name' - prefixed = connection._table_name(name) - self.assertEqual(prefixed, name) - - def test_table_factory(self): - from gcloud_bigtable.happybase.table import Table - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - table = connection.table(name) - - self.assertTrue(isinstance(table, Table)) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - def _table_factory_prefix_helper(self, use_prefix=True): - from gcloud_bigtable.happybase.table import Table - - cluster = _Cluster() # Avoid implicit environ check. - table_prefix = 'table-prefix' - table_prefix_separator = '<>' - connection = self._makeOne( - autoconnect=False, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - name = 'table-name' - table = connection.table(name, use_prefix=use_prefix) - - self.assertTrue(isinstance(table, Table)) - prefixed_name = table_prefix + table_prefix_separator + name - if use_prefix: - self.assertEqual(table.name, prefixed_name) - else: - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - def test_table_factory_with_prefix(self): - self._table_factory_prefix_helper(use_prefix=True) - - def test_table_factory_with_ignored_prefix(self): - self._table_factory_prefix_helper(use_prefix=False) - - def test_tables(self): - from gcloud_bigtable.table import Table - - table_name1 = 'table-name1' - table_name2 = 'table-name2' - cluster = _Cluster(list_tables_result=[ - Table(table_name1, None), - Table(table_name2, None), - ]) - connection = self._makeOne(autoconnect=False, cluster=cluster) - result = connection.tables() - self.assertEqual(result, [table_name1, table_name2]) - - def test_tables_with_prefix(self): - from gcloud_bigtable.table import Table - - table_prefix = 'prefix' - table_prefix_separator = '<>' - unprefixed_table_name1 = 'table-name1' - - table_name1 = (table_prefix + table_prefix_separator + - unprefixed_table_name1) - table_name2 = 'table-name2' - cluster = _Cluster(list_tables_result=[ - Table(table_name1, None), - Table(table_name2, None), - ]) - connection = self._makeOne( - autoconnect=False, cluster=cluster, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator) - result = connection.tables() - self.assertEqual(result, [unprefixed_table_name1]) - - def test_create_table(self): - import operator - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - mock_gc_rule = object() - mock_parse_family_option = _MockCalled(mock_gc_rule) - - name = 'table-name' - col_fam1 = 'cf1' - col_fam_option1 = object() - col_fam2 = u'cf2' - col_fam_option2 = object() - col_fam3 = b'cf3' - col_fam_option3 = object() - families = { - col_fam1: col_fam_option1, - # A trailing colon is also allowed. - col_fam2 + u':': col_fam_option2, - col_fam3 + b':': col_fam_option3, - } - table_instances = [] - col_fam_instances = [] - with _Monkey(MUT, _LowLevelTable=_MockLowLevelTable, - _parse_family_option=mock_parse_family_option): - _MockLowLevelTable._instances = table_instances - _MockLowLevelColumnFamily._instances = col_fam_instances - connection.create_table(name, families) - - # Just one table would have been created. - table_instance, = table_instances - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - self.assertEqual(table_instance.create_calls, 1) - - # Check if our mock was called twice, but we don't know the order. - mock_called = mock_parse_family_option.called_args - self.assertEqual(len(mock_called), 3) - self.assertEqual([len(args) for args in mock_called], [1, 1, 1]) - self.assertEqual( - set(mock_called[0] + mock_called[1] + mock_called[2]), - set([col_fam_option1, col_fam_option2, col_fam_option3])) - - # We expect two column family instances created, but don't know the - # order due to non-deterministic dict.items(). - col_fam_instances.sort(key=operator.attrgetter('column_family_id')) - self.assertEqual(col_fam_instances[0].column_family_id, col_fam1) - self.assertEqual(col_fam_instances[0].gc_rule, mock_gc_rule) - self.assertEqual(col_fam_instances[0].create_calls, 1) - self.assertEqual(col_fam_instances[1].column_family_id, col_fam2) - self.assertEqual(col_fam_instances[1].gc_rule, mock_gc_rule) - self.assertEqual(col_fam_instances[1].create_calls, 1) - self.assertEqual(col_fam_instances[2].column_family_id, col_fam3) - self.assertEqual(col_fam_instances[2].gc_rule, mock_gc_rule) - self.assertEqual(col_fam_instances[2].create_calls, 1) - - def test_create_table_bad_type(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - families = None - with self.assertRaises(TypeError): - connection.create_table(name, families) - - def test_create_table_bad_value(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - families = {} - with self.assertRaises(ValueError): - connection.create_table(name, families) - - def test_delete_table(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - instances = [] - with _Monkey(MUT, _LowLevelTable=_MockLowLevelTable): - _MockLowLevelTable._instances = instances - connection.delete_table(name) - - # Just one table would have been created. - table_instance, = instances - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - self.assertEqual(table_instance.delete_calls, 1) - - def test_delete_table_disable(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - name = 'table-name' - with self.assertRaises(ValueError): - connection.delete_table(name, disable=True) - - def test_enable_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.enable_table(name) - - def test_disable_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.disable_table(name) - - def test_is_table_enabled(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.is_table_enabled(name) - - def test_compact_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - major = True - with self.assertRaises(NotImplementedError): - connection.compact_table(name, major=major) - - -class _Client(object): - - def __init__(self, *args, **kwargs): - self.clusters = kwargs.pop('clusters', []) - for cluster in self.clusters: - cluster.client = self - self.failed_zones = kwargs.pop('failed_zones', []) - self.args = args - self.kwargs = kwargs - self.start_calls = 0 - self.stop_calls = 0 - - def start(self): - self.start_calls += 1 - - def stop(self): - self.stop_calls += 1 - - def list_clusters(self): - return self.clusters, self.failed_zones - - -class _Cluster(object): - - def __init__(self, copies=(), list_tables_result=()): - self.copies = list(copies) - # Included to support Connection.__del__ - self._client = _Client() - self.list_tables_result = list_tables_result - - def copy(self): - if self.copies: - result = self.copies[0] - self.copies[:] = self.copies[1:] - return result - else: - return self - - def list_tables(self): - return self.list_tables_result - - -class _MockLowLevelTable(object): - - _instances = [] - - def __init__(self, *args, **kwargs): - self._instances.append(self) - self.args = args - self.kwargs = kwargs - self.delete_calls = 0 - self.create_calls = 0 - - def delete(self): - self.delete_calls += 1 - - def create(self): - self.create_calls += 1 - - def column_family(self, column_family_id, gc_rule=None): - return _MockLowLevelColumnFamily(column_family_id, gc_rule=gc_rule) - - -class _MockLowLevelColumnFamily(object): - - _instances = [] - - def __init__(self, column_family_id, gc_rule=None): - self._instances.append(self) - self.column_family_id = column_family_id - self.gc_rule = gc_rule - self.create_calls = 0 - - def create(self): - self.create_calls += 1 diff --git a/gcloud_bigtable/happybase/test_pool.py b/gcloud_bigtable/happybase/test_pool.py deleted file mode 100644 index d6142e7..0000000 --- a/gcloud_bigtable/happybase/test_pool.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestConnectionPool(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.pool import ConnectionPool - return ConnectionPool - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - import six - import threading - from gcloud_bigtable.happybase.connection import Connection - - size = 11 - cluster_copy = _Cluster() - all_copies = [cluster_copy] * size - cluster = _Cluster(copies=all_copies) # Avoid implicit environ check. - pool = self._makeOne(size, cluster=cluster) - - self.assertTrue(isinstance(pool._lock, type(threading.Lock()))) - self.assertTrue(isinstance(pool._thread_connections, threading.local)) - self.assertEqual(pool._thread_connections.__dict__, {}) - - queue = pool._queue - self.assertTrue(isinstance(queue, six.moves.queue.LifoQueue)) - self.assertTrue(queue.full()) - self.assertEqual(queue.maxsize, size) - for connection in queue.queue: - self.assertTrue(isinstance(connection, Connection)) - self.assertTrue(connection._cluster is cluster_copy) - - def test_constructor_passes_kwargs(self): - table_prefix = 'foo' - table_prefix_separator = '<>' - cluster = _Cluster() # Avoid implicit environ check. - - size = 1 - pool = self._makeOne(size, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - for connection in pool._queue.queue: - self.assertEqual(connection.table_prefix, table_prefix) - self.assertEqual(connection.table_prefix_separator, - table_prefix_separator) - - def test_constructor_ignores_autoconnect(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase.connection import Connection - from gcloud_bigtable.happybase import pool as MUT - - class ConnectionWithOpen(Connection): - - _open_called = False - - def open(self): - self._open_called = True - - # First make sure the custom Connection class does as expected. - cluster_copy1 = _Cluster() - cluster_copy2 = _Cluster() - cluster_copy3 = _Cluster() - cluster = _Cluster( - copies=[cluster_copy1, cluster_copy2, cluster_copy3]) - connection = ConnectionWithOpen(autoconnect=False, cluster=cluster) - self.assertFalse(connection._open_called) - self.assertTrue(connection._cluster is cluster_copy1) - connection = ConnectionWithOpen(autoconnect=True, cluster=cluster) - self.assertTrue(connection._open_called) - self.assertTrue(connection._cluster is cluster_copy2) - - # Then make sure autoconnect=True is ignored in a pool. - size = 1 - with _Monkey(MUT, Connection=ConnectionWithOpen): - pool = self._makeOne(size, autoconnect=True, cluster=cluster) - - for connection in pool._queue.queue: - self.assertTrue(isinstance(connection, ConnectionWithOpen)) - self.assertTrue(connection._cluster is cluster_copy3) - self.assertFalse(connection._open_called) - - def test_constructor_infers_cluster(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase.connection import Connection - from gcloud_bigtable.happybase import pool as MUT - - size = 1 - cluster_copy = _Cluster() - all_copies = [cluster_copy] * size - cluster = _Cluster(copies=all_copies) - - mock_get_cluster = _MockCalled(cluster) - with _Monkey(MUT, _get_cluster=mock_get_cluster): - pool = self._makeOne(size) - - for connection in pool._queue.queue: - self.assertTrue(isinstance(connection, Connection)) - # We know that the Connection() constructor will - # call cluster.copy(). - self.assertTrue(connection._cluster is cluster_copy) - - mock_get_cluster.check_called(self, [()], [{'timeout': None}]) - - def test_constructor_non_integer_size(self): - size = None - with self.assertRaises(TypeError): - self._makeOne(size) - - def test_constructor_non_positive_size(self): - size = -10 - with self.assertRaises(ValueError): - self._makeOne(size) - size = 0 - with self.assertRaises(ValueError): - self._makeOne(size) - - def _makeOneWithMockQueue(self, queue_return): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import pool as MUT - - # We are going to use a fake queue, so we don't want any connections - # or clusters to be created in the constructor. - size = -1 - cluster = object() - with _Monkey(MUT, _MIN_POOL_SIZE=size): - pool = self._makeOne(size, cluster=cluster) - - pool._queue = _Queue(queue_return) - return pool - - def test__acquire_connection(self): - queue_return = object() - pool = self._makeOneWithMockQueue(queue_return) - - timeout = 432 - connection = pool._acquire_connection(timeout=timeout) - self.assertTrue(connection is queue_return) - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, []) - - def test__acquire_connection_failure(self): - from gcloud_bigtable.happybase.pool import NoConnectionsAvailable - - pool = self._makeOneWithMockQueue(None) - timeout = 1027 - with self.assertRaises(NoConnectionsAvailable): - pool._acquire_connection(timeout=timeout) - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, []) - - def test_connection_is_context_manager(self): - import contextlib - - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - cnxn_context = pool.connection() - self.assertTrue(isinstance(cnxn_context, - contextlib.GeneratorContextManager)) - - def test_connection_no_current_cnxn(self): - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - timeout = 55 - - self.assertFalse(hasattr(pool._thread_connections, 'current')) - with pool.connection(timeout=timeout) as connection: - self.assertEqual(pool._thread_connections.current, queue_return) - self.assertTrue(connection is queue_return) - self.assertFalse(hasattr(pool._thread_connections, 'current')) - - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, - [(queue_return, None, None)]) - - def test_connection_with_current_cnxn(self): - current_cnxn = _Connection() - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - pool._thread_connections.current = current_cnxn - timeout = 8001 - - with pool.connection(timeout=timeout) as connection: - self.assertTrue(connection is current_cnxn) - - self.assertEqual(pool._queue._get_calls, []) - self.assertEqual(pool._queue._put_calls, []) - self.assertEqual(pool._thread_connections.current, current_cnxn) - - -class _Client(object): - - def __init__(self): - self.stop_calls = 0 - - def stop(self): - self.stop_calls += 1 - - -class _Connection(object): - - def open(self): - pass - - -class _Cluster(object): - - def __init__(self, copies=()): - self.copies = list(copies) - # Included to support Connection.__del__ - self._client = _Client() - - def copy(self): - if self.copies: - result = self.copies[0] - self.copies[:] = self.copies[1:] - return result - else: - return self - - -class _Queue(object): - - def __init__(self, result=None): - self.result = result - self._get_calls = [] - self._put_calls = [] - - def get(self, block=None, timeout=None): - self._get_calls.append((block, timeout)) - if self.result is None: - import six - raise six.moves.queue.Empty - else: - return self.result - - def put(self, item, block=None, timeout=None): - self._put_calls.append((item, block, timeout)) diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py deleted file mode 100644 index 1b8d278..0000000 --- a/gcloud_bigtable/happybase/test_table.py +++ /dev/null @@ -1,1363 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest2 - - -class Test_make_row(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import make_row - return make_row(*args, **kwargs) - - def test_it(self): - with self.assertRaises(NotImplementedError): - self._callFUT({}, False) - - -class Test_make_ordered_row(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import make_ordered_row - return make_ordered_row(*args, **kwargs) - - def test_it(self): - with self.assertRaises(NotImplementedError): - self._callFUT([], False) - - -class Test__gc_rule_to_dict(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _gc_rule_to_dict - return _gc_rule_to_dict(*args, **kwargs) - - def test_with_null(self): - gc_rule = None - result = self._callFUT(gc_rule) - self.assertEqual(result, {}) - - def test_with_max_versions(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - - max_versions = 2 - gc_rule = MaxVersionsGCRule(max_versions) - result = self._callFUT(gc_rule) - expected_result = {'max_versions': max_versions} - self.assertEqual(result, expected_result) - - def test_with_max_age(self): - import datetime - from gcloud_bigtable.column_family import MaxAgeGCRule - - time_to_live = 101 - max_age = datetime.timedelta(seconds=time_to_live) - gc_rule = MaxAgeGCRule(max_age) - result = self._callFUT(gc_rule) - expected_result = {'time_to_live': time_to_live} - self.assertEqual(result, expected_result) - - def test_with_non_gc_rule(self): - gc_rule = object() - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_gc_rule_union(self): - from gcloud_bigtable.column_family import GCRuleUnion - - gc_rule = GCRuleUnion(rules=[]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_other_than_two(self): - from gcloud_bigtable.column_family import GCRuleIntersection - - gc_rule = GCRuleIntersection(rules=[]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_two_max_num_versions(self): - from gcloud_bigtable.column_family import GCRuleIntersection - from gcloud_bigtable.column_family import MaxVersionsGCRule - - rule1 = MaxVersionsGCRule(1) - rule2 = MaxVersionsGCRule(2) - gc_rule = GCRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_two_rules(self): - import datetime - from gcloud_bigtable.column_family import GCRuleIntersection - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - time_to_live = 101 - max_age = datetime.timedelta(seconds=time_to_live) - rule1 = MaxAgeGCRule(max_age) - max_versions = 2 - rule2 = MaxVersionsGCRule(max_versions) - gc_rule = GCRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - expected_result = { - 'max_versions': max_versions, - 'time_to_live': time_to_live, - } - self.assertEqual(result, expected_result) - - def test_with_intersection_two_nested_rules(self): - from gcloud_bigtable.column_family import GCRuleIntersection - - rule1 = GCRuleIntersection(rules=[]) - rule2 = GCRuleIntersection(rules=[]) - gc_rule = GCRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - -class Test__convert_to_time_range(unittest2.TestCase): - - def _callFUT(self, timestamp=None): - from gcloud_bigtable.happybase.table import _convert_to_time_range - return _convert_to_time_range(timestamp=timestamp) - - def test_null(self): - timestamp = None - result = self._callFUT(timestamp=timestamp) - self.assertEqual(result, None) - - def test_invalid_type(self): - timestamp = object() - with self.assertRaises(TypeError): - self._callFUT(timestamp=timestamp) - - def test_success(self): - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - from gcloud_bigtable.row import TimestampRange - - timestamp = 1441928298571 - ts_dt = _microseconds_to_timestamp(1000 * timestamp) - result = self._callFUT(timestamp=timestamp) - self.assertTrue(isinstance(result, TimestampRange)) - self.assertEqual(result.start, None) - self.assertEqual(result.end, ts_dt) - - -class Test__cells_to_pairs(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _cells_to_pairs - return _cells_to_pairs(*args, **kwargs) - - def test_without_timestamp(self): - from gcloud_bigtable.row_data import Cell - - value1 = 'foo' - cell1 = Cell(value=value1, timestamp=None) - value2 = 'bar' - cell2 = Cell(value=value2, timestamp=None) - - result = self._callFUT([cell1, cell2]) - self.assertEqual(result, [value1, value2]) - - def test_with_timestamp(self): - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - from gcloud_bigtable.row_data import Cell - - value1 = 'foo' - ts1_millis = 1221934570148 - ts1 = _microseconds_to_timestamp(ts1_millis * 1000) - cell1 = Cell(value=value1, timestamp=ts1) - - value2 = 'bar' - ts2_millis = 1221955575548 - ts2 = _microseconds_to_timestamp(ts2_millis * 1000) - cell2 = Cell(value=value2, timestamp=ts2) - - result = self._callFUT([cell1, cell2], include_timestamp=True) - self.assertEqual(result, - [(value1, ts1_millis), (value2, ts2_millis)]) - - -class Test__filter_chain_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _filter_chain_helper - return _filter_chain_helper(*args, **kwargs) - - def test_no_filters(self): - with self.assertRaises(ValueError): - self._callFUT() - - def test_single_filter(self): - from gcloud_bigtable.row import CellsColumnLimitFilter - - versions = 1337 - result = self._callFUT(versions=versions) - self.assertTrue(isinstance(result, CellsColumnLimitFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(result.num_cells, versions) - - def test_existing_filters(self): - from gcloud_bigtable.row import CellsColumnLimitFilter - - filters = [] - versions = 1337 - result = self._callFUT(versions=versions, filters=filters) - # Make sure filters has grown. - self.assertEqual(filters, [result]) - - self.assertTrue(isinstance(result, CellsColumnLimitFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(result.num_cells, versions) - - def _column_helper(self, num_filters, versions=None, timestamp=None): - from gcloud_bigtable.row import ColumnQualifierRegexFilter - from gcloud_bigtable.row import FamilyNameRegexFilter - from gcloud_bigtable.row import RowFilterChain - - col_fam = 'cf1' - qual = 'qual' - column = col_fam + ':' + qual - result = self._callFUT(column, versions=versions, timestamp=timestamp) - self.assertTrue(isinstance(result, RowFilterChain)) - - self.assertEqual(len(result.filters), num_filters) - fam_filter = result.filters[0] - qual_filter = result.filters[1] - self.assertTrue(isinstance(fam_filter, FamilyNameRegexFilter)) - self.assertTrue(isinstance(qual_filter, ColumnQualifierRegexFilter)) - - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(fam_filter.regex, col_fam) - self.assertEqual(qual_filter.regex, qual) - - return result - - def test_column_only(self): - self._column_helper(num_filters=2) - - def test_with_versions(self): - from gcloud_bigtable.row import CellsColumnLimitFilter - - versions = 11 - result = self._column_helper(num_filters=3, versions=versions) - - version_filter = result.filters[2] - self.assertTrue(isinstance(version_filter, CellsColumnLimitFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(version_filter.num_cells, versions) - - def test_with_timestamp(self): - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - from gcloud_bigtable.row import TimestampRange - from gcloud_bigtable.row import TimestampRangeFilter - - timestamp = 1441928298571 - result = self._column_helper(num_filters=3, timestamp=timestamp) - - range_filter = result.filters[2] - self.assertTrue(isinstance(range_filter, TimestampRangeFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - time_range = range_filter.range_ - self.assertTrue(isinstance(time_range, TimestampRange)) - self.assertEqual(time_range.start, None) - ts_dt = _microseconds_to_timestamp(1000 * timestamp) - self.assertEqual(time_range.end, ts_dt) - - def test_with_all_options(self): - versions = 11 - timestamp = 1441928298571 - self._column_helper(num_filters=4, versions=versions, - timestamp=timestamp) - - -class Test__columns_filter_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _columns_filter_helper - return _columns_filter_helper(*args, **kwargs) - - def test_no_columns(self): - columns = [] - with self.assertRaises(ValueError): - self._callFUT(columns) - - def test_single_column(self): - from gcloud_bigtable.row import FamilyNameRegexFilter - - col_fam = 'cf1' - columns = [col_fam] - result = self._callFUT(columns) - expected_result = FamilyNameRegexFilter(col_fam) - self.assertEqual(result, expected_result) - - def test_column_and_column_families(self): - from gcloud_bigtable.row import ColumnQualifierRegexFilter - from gcloud_bigtable.row import FamilyNameRegexFilter - from gcloud_bigtable.row import RowFilterChain - from gcloud_bigtable.row import RowFilterUnion - - col_fam1 = 'cf1' - col_fam2 = 'cf2' - col_qual2 = 'qual2' - columns = [col_fam1, col_fam2 + ':' + col_qual2] - result = self._callFUT(columns) - - self.assertTrue(isinstance(result, RowFilterUnion)) - self.assertEqual(len(result.filters), 2) - filter1 = result.filters[0] - filter2 = result.filters[1] - - self.assertTrue(isinstance(filter1, FamilyNameRegexFilter)) - self.assertEqual(filter1.regex, col_fam1) - - self.assertTrue(isinstance(filter2, RowFilterChain)) - filter2a, filter2b = filter2.filters - self.assertTrue(isinstance(filter2a, FamilyNameRegexFilter)) - self.assertEqual(filter2a.regex, col_fam2) - self.assertTrue(isinstance(filter2b, ColumnQualifierRegexFilter)) - self.assertEqual(filter2b.regex, col_qual2) - - -class Test__row_keys_filter_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _row_keys_filter_helper - return _row_keys_filter_helper(*args, **kwargs) - - def test_no_rows(self): - row_keys = [] - with self.assertRaises(ValueError): - self._callFUT(row_keys) - - def test_single_row(self): - from gcloud_bigtable.row import RowKeyRegexFilter - - row_key = b'row-key' - row_keys = [row_key] - result = self._callFUT(row_keys) - expected_result = RowKeyRegexFilter(row_key) - self.assertEqual(result, expected_result) - - def test_many_rows(self): - from gcloud_bigtable.row import RowFilterUnion - from gcloud_bigtable.row import RowKeyRegexFilter - - row_key1 = b'row-key1' - row_key2 = b'row-key2' - row_key3 = b'row-key3' - row_keys = [row_key1, row_key2, row_key3] - result = self._callFUT(row_keys) - - filter1 = RowKeyRegexFilter(row_key1) - filter2 = RowKeyRegexFilter(row_key2) - filter3 = RowKeyRegexFilter(row_key3) - expected_result = RowFilterUnion(filters=[filter1, filter2, filter3]) - self.assertEqual(result, expected_result) - - -class Test__string_successor(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _string_successor - return _string_successor(*args, **kwargs) - - def test_with_alphanumeric(self): - self.assertEqual(self._callFUT(b'boa'), b'bob') - self.assertEqual(self._callFUT(b'abc1'), b'abc2') - - def test_with_last_byte(self): - self.assertEqual(self._callFUT(b'boa\xff'), b'bob') - - def test_with_empty_string(self): - self.assertEqual(self._callFUT(b''), b'') - - def test_with_all_last_bytes(self): - self.assertEqual(self._callFUT(b'\xff\xff\xff'), b'') - - def test_with_unicode_input(self): - self.assertEqual(self._callFUT(u'boa'), b'bob') - - -class TestTable(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.table import Table - return Table - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - cluster = object() - connection = _Connection(cluster) - tables_constructed = [] - - def make_low_level_table(*args, **kwargs): - result = _MockLowLevelTable(*args, **kwargs) - tables_constructed.append(result) - return result - - with _Monkey(MUT, _LowLevelTable=make_low_level_table): - table = self._makeOne(name, connection) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - table_instance, = tables_constructed - self.assertEqual(table._low_level_table, table_instance) - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - - def test_constructor_null_connection(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - self.assertEqual(table._low_level_table, None) - - def test___repr__(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - self.assertEqual(repr(table), '') - - def test_families(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - - # Mock the column families to be returned. - col_fam_name = 'fam' - gc_rule = object() - col_fam = _MockLowLevelColumnFamily(col_fam_name, gc_rule=gc_rule) - col_fams = {col_fam_name: col_fam} - table._low_level_table.column_families = col_fams - - to_dict_result = object() - mock_gc_rule_to_dict = _MockCalled(to_dict_result) - with _Monkey(MUT, _gc_rule_to_dict=mock_gc_rule_to_dict): - result = table.families() - - self.assertEqual(result, {col_fam_name: to_dict_result}) - self.assertEqual(table._low_level_table.list_column_families_calls, 1) - - # Check the input to our mock. - mock_gc_rule_to_dict.check_called(self, [(gc_rule,)]) - - def test_regions(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - with self.assertRaises(NotImplementedError): - table.regions() - - def test_row_empty_row(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - timestamp = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper): - result = table.row(row_key, timestamp=timestamp) - - # read_row_result == None --> No results. - self.assertEqual(result, {}) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'filters': [], - 'versions': 1, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_row_with_columns(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - columns = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.row(row_key, columns=columns) - - # read_row_result == None --> No results. - self.assertEqual(result, {}) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - mock_columns_filter_helper.check_called(self, [(columns,)]) - expected_kwargs = { - 'filters': [fake_col_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_row_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key = 'row-key' - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - partial_row = PartialRowData(row_key) - table._low_level_table.read_row_result = partial_row - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_pair = object() - mock_cells_to_pairs = _MockCalled([fake_pair]) - - col_fam = u'cf1' - qual = b'qual' - fake_cells = object() - partial_row._cells = {col_fam: {qual: fake_cells}} - include_timestamp = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.row(row_key, include_timestamp=include_timestamp) - - # The results come from _cells_to_pairs. - expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} - self.assertEqual(result, expected_result) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'filters': [], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_rows_empty_row(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - result = table.rows([]) - self.assertEqual(result, []) - - def test_rows_with_columns(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - rr_result = _MockPartialRowsData() - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_all_calls, 0) - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_rows_filter = object() - mock_row_keys_filter_helper = _MockCalled(fake_rows_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - rows = ['row-key'] - columns = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _row_keys_filter_helper=mock_row_keys_filter_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.rows(rows, columns=columns) - - # read_rows_result == Empty PartialRowsData --> No results. - self.assertEqual(result, []) - - read_rows_args = () - read_rows_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_all_calls, 1) - - mock_columns_filter_helper.check_called(self, [(columns,)]) - mock_row_keys_filter_helper.check_called(self, [(rows,)]) - expected_kwargs = { - 'filters': [fake_col_filter, fake_rows_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_rows_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key1 = 'row-key1' - row_key2 = 'row-key2' - rows = [row_key1, row_key2] - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - - row1 = PartialRowData(row_key1) - # Return row1 but not row2 - rr_result = _MockPartialRowsData(rows={row_key1: row1}) - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_all_calls, 0) - - fake_rows_filter = object() - mock_row_keys_filter_helper = _MockCalled(fake_rows_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_pair = object() - mock_cells_to_pairs = _MockCalled([fake_pair]) - - col_fam = u'cf1' - qual = b'qual' - fake_cells = object() - row1._cells = {col_fam: {qual: fake_cells}} - include_timestamp = object() - with _Monkey(MUT, _row_keys_filter_helper=mock_row_keys_filter_helper, - _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.rows(rows, include_timestamp=include_timestamp) - - # read_rows_result == PartialRowsData with row_key1 - expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} - self.assertEqual(result, [(row_key1, expected_result)]) - - read_rows_args = () - read_rows_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_all_calls, 1) - - mock_row_keys_filter_helper.check_called(self, [(rows,)]) - expected_kwargs = { - 'filters': [fake_rows_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_cells_empty_row(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - column = 'fam:col1' - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper): - result = table.cells(row_key, column) - - # read_row_result == None --> No results. - self.assertEqual(result, []) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'column': column, - 'versions': None, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_cells_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key = 'row-key' - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - partial_row = PartialRowData(row_key) - table._low_level_table.read_row_result = partial_row - - # These are all passed to mocks. - versions = object() - timestamp = object() - include_timestamp = object() - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_result = object() - mock_cells_to_pairs = _MockCalled(fake_result) - - col_fam = 'cf1' - qual = 'qual' - fake_cells = object() - partial_row._cells = {col_fam: {qual: fake_cells}} - column = col_fam + ':' + qual - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.cells(row_key, column, versions=versions, - timestamp=timestamp, - include_timestamp=include_timestamp) - - self.assertEqual(result, fake_result) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - filter_kwargs = { - 'column': column, - 'versions': versions, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [filter_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_scan_with_batch_size(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - # Use unknown to force a TypeError, so we don't need to - # stub out the rest of the method. - with self.assertRaises(TypeError): - with _Monkey(MUT, _WARN=mock_warn): - list(table.scan(batch_size=object(), unknown=None)) - - self.assertEqual(len(warned), 1) - self.assertIn('batch_size', warned[0]) - - def test_scan_with_scan_batching(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - # Use unknown to force a TypeError, so we don't need to - # stub out the rest of the method. - with self.assertRaises(TypeError): - with _Monkey(MUT, _WARN=mock_warn): - list(table.scan(scan_batching=object(), unknown=None)) - - self.assertEqual(len(warned), 1) - self.assertIn('scan_batching', warned[0]) - - def test_scan_with_sorted_columns(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - warned = [] - - def mock_warn(msg): - warned.append(msg) - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - # Use unknown to force a TypeError, so we don't need to - # stub out the rest of the method. - with self.assertRaises(TypeError): - with _Monkey(MUT, _WARN=mock_warn): - list(table.scan(sorted_columns=object(), unknown=None)) - - self.assertEqual(len(warned), 1) - self.assertIn('sorted_columns', warned[0]) - - def test_scan_with_invalid_limit(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(limit=-10)) - - def test_scan_with_row_prefix_and_row_start(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(row_prefix='a', row_stop='abc')) - - def test_scan_with_string_filter(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(TypeError): - list(table.scan(filter='some-string')) - - def _scan_test_helper(self, row_start=None, row_stop=None, row_prefix=None, - columns=None, filter_=None, timestamp=None, - include_timestamp=False, limit=None, rr_result=None, - expected_result=None): - import types - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - rr_result = rr_result or _MockPartialRowsData() - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_next_calls, 0) - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.scan(row_start=row_start, row_stop=row_stop, - row_prefix=row_prefix, columns=columns, - filter=filter_, timestamp=timestamp, - include_timestamp=include_timestamp, - limit=limit) - self.assertTrue(isinstance(result, types.GeneratorType)) - # Need to consume the result while the monkey patch is applied. - # read_rows_result == Empty PartialRowsData --> No results. - expected_result = expected_result or [] - self.assertEqual(list(result), expected_result) - - read_rows_args = () - if row_prefix: - row_start = row_prefix - row_stop = MUT._string_successor(row_prefix) - read_rows_kwargs = { - 'end_key': row_stop, - 'filter_': fake_filter, - 'limit': limit, - 'start_key': row_start, - } - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_next_calls, - rr_result.iterations + 1) - - if columns is not None: - mock_columns_filter_helper.check_called(self, [(columns,)]) - else: - mock_columns_filter_helper.check_called(self, []) - - filters = [] - if filter_ is not None: - filters.append(filter_) - if columns: - filters.append(fake_col_filter) - expected_kwargs = { - 'filters': filters, - 'versions': 1, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_scan_with_columns(self): - columns = object() - self._scan_test_helper(columns=columns) - - def test_scan_with_row_start_and_stop(self): - row_start = 'bar' - row_stop = 'foo' - self._scan_test_helper(row_start=row_start, row_stop=row_stop) - - def test_scan_with_row_prefix(self): - row_prefix = 'row-prefi' - self._scan_test_helper(row_prefix=row_prefix) - - def test_scan_with_filter(self): - mock_filter = object() - self._scan_test_helper(filter_=mock_filter) - - def test_scan_with_no_results(self): - limit = 1337 - timestamp = object() - self._scan_test_helper(timestamp=timestamp, limit=limit) - - def test_scan_with_results(self): - from gcloud_bigtable.row_data import PartialRowData - - row_key1 = 'row-key1' - row1 = PartialRowData(row_key1) - rr_result = _MockPartialRowsData(rows={row_key1: row1}, iterations=1) - - include_timestamp = object() - expected_result = [(row_key1, {})] - self._scan_test_helper(include_timestamp=include_timestamp, - rr_result=rr_result, - expected_result=expected_result) - - def test_put(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.happybase.table import _WAL_SENTINEL - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - batches_created = [] - - def make_batch(*args, **kwargs): - result = _MockBatch(*args, **kwargs) - batches_created.append(result) - return result - - row = 'row-key' - data = {'fam:col': 'foo'} - timestamp = None - with _Monkey(MUT, Batch=make_batch): - result = table.put(row, data, timestamp=timestamp) - - # There is no return value. - self.assertEqual(result, None) - - # Check how the batch was created and used. - batch, = batches_created - self.assertTrue(isinstance(batch, _MockBatch)) - self.assertEqual(batch.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': None, - 'transaction': False, - 'wal': _WAL_SENTINEL, - } - self.assertEqual(batch.kwargs, expected_kwargs) - # Make sure it was a successful context manager - self.assertEqual(batch.exit_vals, [(None, None, None)]) - self.assertEqual(batch.put_args, [(row, data)]) - self.assertEqual(batch.delete_args, []) - - def test_delete(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.happybase.table import _WAL_SENTINEL - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - batches_created = [] - - def make_batch(*args, **kwargs): - result = _MockBatch(*args, **kwargs) - batches_created.append(result) - return result - - row = 'row-key' - columns = ['fam:col1', 'fam:col2'] - timestamp = None - with _Monkey(MUT, Batch=make_batch): - result = table.delete(row, columns=columns, timestamp=timestamp) - - # There is no return value. - self.assertEqual(result, None) - - # Check how the batch was created and used. - batch, = batches_created - self.assertTrue(isinstance(batch, _MockBatch)) - self.assertEqual(batch.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': None, - 'transaction': False, - 'wal': _WAL_SENTINEL, - } - self.assertEqual(batch.kwargs, expected_kwargs) - # Make sure it was a successful context manager - self.assertEqual(batch.exit_vals, [(None, None, None)]) - self.assertEqual(batch.put_args, []) - self.assertEqual(batch.delete_args, [(row, columns)]) - - def test_batch(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - timestamp = object() - batch_size = 42 - transaction = False # Must be False when batch_size is non-null - wal = object() - - with _Monkey(MUT, Batch=_MockBatch): - result = table.batch(timestamp=timestamp, batch_size=batch_size, - transaction=transaction, wal=wal) - - self.assertTrue(isinstance(result, _MockBatch)) - self.assertEqual(result.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': batch_size, - 'transaction': transaction, - 'wal': wal, - } - self.assertEqual(result.kwargs, expected_kwargs) - - def test_counter_get(self): - klass = self._getTargetClass() - counter_value = 1337 - - class TableWithInc(klass): - - incremented = [] - value = counter_value - - def counter_inc(self, row, column, value=1): - self.incremented.append((row, column, value)) - self.value += value - return self.value - - name = 'table-name' - connection = None - table = TableWithInc(name, connection) - - row = 'row-key' - column = 'fam:col1' - self.assertEqual(TableWithInc.incremented, []) - result = table.counter_get(row, column) - self.assertEqual(result, counter_value) - self.assertEqual(TableWithInc.incremented, [(row, column, 0)]) - - def test_counter_set(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - row = 'row-key' - column = 'fam:col1' - value = 42 - with self.assertRaises(NotImplementedError): - table.counter_set(row, column, value=value) - - def test_counter_dec(self): - klass = self._getTargetClass() - counter_value = 42 - - class TableWithInc(klass): - - incremented = [] - value = counter_value - - def counter_inc(self, row, column, value=1): - self.incremented.append((row, column, value)) - self.value += value - return self.value - - name = 'table-name' - connection = None - table = TableWithInc(name, connection) - - row = 'row-key' - column = 'fam:col1' - dec_value = 987 - self.assertEqual(TableWithInc.incremented, []) - result = table.counter_dec(row, column, value=dec_value) - self.assertEqual(result, counter_value - dec_value) - self.assertEqual(TableWithInc.incremented, [(row, column, -dec_value)]) - - def _counter_inc_helper(self, row, column, value, commit_result): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - # Mock the return values. - table._low_level_table = _MockLowLevelTable() - table._low_level_table.row_values[row] = _MockLowLevelRow( - row, commit_result=commit_result) - - result = table.counter_inc(row, column, value=value) - - incremented_value = value + _MockLowLevelRow.COUNTER_DEFAULT - self.assertEqual(result, incremented_value) - - # Check the row values returned. - row_obj = table._low_level_table.row_values[row] - self.assertEqual(row_obj.counts, - {tuple(column.split(':')): incremented_value}) - - def test_counter_inc(self): - import struct - - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - packed_value = struct.pack('>q', value) - fake_timestamp = None - commit_result = { - col_fam: { - col_qual: [(packed_value, fake_timestamp)], - } - } - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_unicode(self): - import struct - - row = 'row-key' - col_fam = b'fam' - col_qual = b'col1' - column = (col_fam + b':' + col_qual).decode('utf-8') - value = 42 - packed_value = struct.pack('>q', value) - fake_timestamp = None - commit_result = { - col_fam: { - col_qual: [(packed_value, fake_timestamp)], - } - } - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_bad_result(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = None - with self.assertRaises(TypeError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_key_error(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = {} - with self.assertRaises(KeyError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_nested_key_error(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = {col_fam: {}} - with self.assertRaises(KeyError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_non_unique_cell(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - fake_timestamp = None - packed_value = None - commit_result = { - col_fam: { - col_qual: [ - (packed_value, fake_timestamp), - (packed_value, fake_timestamp), - ], - } - } - with self.assertRaises(ValueError): - self._counter_inc_helper(row, column, value, commit_result) - - -class _MockLowLevelTable(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.list_column_families_calls = 0 - self.column_families = {} - self.row_values = {} - self.read_row_calls = [] - self.read_row_result = None - self.read_rows_calls = [] - self.read_rows_result = None - - def list_column_families(self): - self.list_column_families_calls += 1 - return self.column_families - - def row(self, row_key): - return self.row_values[row_key] - - def read_row(self, *args, **kwargs): - self.read_row_calls.append((args, kwargs)) - return self.read_row_result - - def read_rows(self, *args, **kwargs): - self.read_rows_calls.append((args, kwargs)) - return self.read_rows_result - - -class _MockLowLevelColumnFamily(object): - - def __init__(self, column_family_id, gc_rule=None): - self.column_family_id = column_family_id - self.gc_rule = gc_rule - - -class _Connection(object): - - def __init__(self, cluster): - self._cluster = cluster - - -class _MockBatch(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.exit_vals = [] - self.put_args = [] - self.delete_args = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.exit_vals.append((exc_type, exc_value, traceback)) - - def put(self, *args): - self.put_args.append(args) - - def delete(self, *args): - self.delete_args.append(args) - - -class _MockLowLevelRow(object): - - COUNTER_DEFAULT = 0 - - def __init__(self, row_key, commit_result=None): - self.row_key = row_key - self.counts = {} - self.commit_result = commit_result - - def increment_cell_value(self, column_family_id, column, int_value): - count = self.counts.setdefault((column_family_id, column), - self.COUNTER_DEFAULT) - self.counts[(column_family_id, column)] = count + int_value - - def commit_modifications(self): - return self.commit_result - - -class _MockPartialRowsData(object): - - def __init__(self, rows=None, iterations=0): - self.rows = rows or {} - self.consume_all_calls = 0 - self.consume_next_calls = 0 - self.iterations = iterations - - def consume_all(self): - self.consume_all_calls += 1 - - def consume_next(self): - self.consume_next_calls += 1 - if self.consume_next_calls > self.iterations: - raise StopIteration diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py deleted file mode 100644 index 3755b89..0000000 --- a/gcloud_bigtable/row.py +++ /dev/null @@ -1,1339 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Row.""" - - -import struct - -import six - -from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds -from gcloud_bigtable._non_upstream_helpers import _to_bytes - - -# pylint: disable=invalid-name -# Alias for easy upstream diffs. -_microseconds_from_datetime = _timestamp_to_microseconds -_datetime_from_microseconds = _microseconds_to_timestamp -# pylint: enable=invalid-name -_MAX_MUTATIONS = 100000 -_PACK_I64 = struct.Struct('>q').pack - - -class Row(object): - """Representation of a Google Cloud Bigtable Row. - - .. note:: - - A :class:`Row` accumulates mutations locally via the :meth:`set_cell`, - :meth:`delete`, :meth:`delete_cell` and :meth:`delete_cells` methods. - To actually send these mutations to the Google Cloud Bigtable API, you - must call :meth:`commit`. If a ``filter_`` is set on the :class:`Row`, - the mutations must have an associated state: :data:`True` or - :data:`False`. The mutations will be applied conditionally, based on - whether the filter matches any cells in the :class:`Row` or not. - - :type row_key: bytes - :param row_key: The key for the current row. - - :type table: :class:`Table ` - :param table: The table that owns the row. - - :type filter_: :class:`RowFilter` - :param filter_: (Optional) Filter to be used for conditional mutations. - If a filter is set, then the :class:`Row` will accumulate - mutations for either a :data:`True` or :data:`False` state. - When :meth:`commit`-ed, the mutations for the :data:`True` - state will be applied if the filter matches any cells in - the row, otherwise the :data:`False` state will be. - """ - - ALL_COLUMNS = object() - """Sentinel value used to indicate all columns in a column family.""" - - def __init__(self, row_key, table, filter_=None): - self._row_key = _to_bytes(row_key) - self._table = table - self._filter = filter_ - self._rule_pb_list = [] - if self._filter is None: - self._pb_mutations = [] - self._true_pb_mutations = None - self._false_pb_mutations = None - else: - self._pb_mutations = None - self._true_pb_mutations = [] - self._false_pb_mutations = [] - - def _get_mutations(self, state=None): - """Gets the list of mutations for a given state. - - If the state is :data`None` but there is a filter set, then we've - reached an invalid state. Similarly if no filter is set but the - state is not :data:`None`. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - - :rtype: list - :returns: The list to add new mutations to (for the current state). - :raises: :class:`ValueError ` - """ - if state is None: - if self._filter is not None: - raise ValueError('A filter is set on the current row, but no ' - 'state given for the mutation') - return self._pb_mutations - else: - if self._filter is None: - raise ValueError('No filter was set on the current row, but a ' - 'state was given for the mutation') - if state: - return self._true_pb_mutations - else: - return self._false_pb_mutations - - def set_cell(self, column_family_id, column, value, timestamp=None, - state=None): - """Sets a value in this row. - - The cell is determined by the ``row_key`` of the :class:`Row` and the - ``column``. The ``column`` must be in an existing - :class:`.column_family.ColumnFamily` (as determined by - ``column_family_id``). - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type value: bytes or :class:`int` - :param value: The value to set in the cell. If an integer is used, - will be interpreted as a 64-bit big-endian signed - integer (8 bytes). - - :type timestamp: :class:`datetime.datetime` - :param timestamp: (Optional) The timestamp of the operation. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - column = _to_bytes(column) - if isinstance(value, six.integer_types): - value = _PACK_I64(value) - value = _to_bytes(value) - if timestamp is None: - # Use -1 for current Bigtable server time. - timestamp_micros = -1 - else: - timestamp_micros = _microseconds_from_datetime(timestamp) - # Truncate to millisecond granularity. - timestamp_micros -= (timestamp_micros % 1000) - - mutation_val = data_pb2.Mutation.SetCell( - family_name=column_family_id, - column_qualifier=column, - timestamp_micros=timestamp_micros, - value=value, - ) - mutation_pb = data_pb2.Mutation(set_cell=mutation_val) - self._get_mutations(state).append(mutation_pb) - - def append_cell_value(self, column_family_id, column, value): - """Appends a value to an existing cell. - - .. note:: - - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type value: bytes - :param value: The value to append to the existing value in the cell. If - the targeted cell is unset, it will be treated as - containing the empty string. - """ - column = _to_bytes(column) - value = _to_bytes(value) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - append_value=value) - self._rule_pb_list.append(rule_pb) - - def increment_cell_value(self, column_family_id, column, int_value): - """Increments a value in an existing cell. - - Assumes the value in the cell is stored as a 64 bit integer - serialized to bytes. - - .. note:: - - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type int_value: int - :param int_value: The value to increment the existing value in the cell - by. If the targeted cell is unset, it will be treated - as containing a zero. Otherwise, the targeted cell - must contain an 8-byte value (interpreted as a 64-bit - big-endian signed integer), or the entire request - will fail. - """ - column = _to_bytes(column) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - increment_amount=int_value) - self._rule_pb_list.append(rule_pb) - - def delete(self, state=None): - """Deletes this row from the table. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - mutation_val = data_pb2.Mutation.DeleteFromRow() - mutation_pb = data_pb2.Mutation(delete_from_row=mutation_val) - self._get_mutations(state).append(mutation_pb) - - def delete_cell(self, column_family_id, column, time_range=None, - state=None): - """Deletes cell in this row. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column - or columns with cells being deleted. Must be - of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family that will have a - cell deleted. - - :type time_range: :class:`TimestampRange` - :param time_range: (Optional) The range of time within which cells - should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - self.delete_cells(column_family_id, [column], time_range=time_range, - state=state) - - def delete_cells(self, column_family_id, columns, time_range=None, - state=None): - """Deletes cells in this row. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column - or columns with cells being deleted. Must be - of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type columns: :class:`list` of :class:`str` / - :func:`unicode `, or :class:`object` - :param columns: The columns within the column family that will have - cells deleted. If :attr:`Row.ALL_COLUMNS` is used then - the entire column family will be deleted from the row. - - :type time_range: :class:`TimestampRange` - :param time_range: (Optional) The range of time within which cells - should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - mutations_list = self._get_mutations(state) - if columns is self.ALL_COLUMNS: - mutation_val = data_pb2.Mutation.DeleteFromFamily( - family_name=column_family_id, - ) - mutation_pb = data_pb2.Mutation(delete_from_family=mutation_val) - mutations_list.append(mutation_pb) - else: - delete_kwargs = {} - if time_range is not None: - delete_kwargs['time_range'] = time_range.to_pb() - - to_append = [] - for column in columns: - column = _to_bytes(column) - # time_range will never change if present, but the rest of - # delete_kwargs will - delete_kwargs.update( - family_name=column_family_id, - column_qualifier=column, - ) - mutation_val = data_pb2.Mutation.DeleteFromColumn( - **delete_kwargs) - mutation_pb = data_pb2.Mutation( - delete_from_column=mutation_val) - to_append.append(mutation_pb) - - # We don't add the mutations until all columns have been - # processed without error. - mutations_list.extend(to_append) - - def _commit_mutate(self): - """Makes a ``MutateRow`` API request. - - Assumes no filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. - - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - mutations_list = self._get_mutations() - num_mutations = len(mutations_list) - if num_mutations == 0: - return - if num_mutations > _MAX_MUTATIONS: - raise ValueError('%d total mutations exceed the maximum allowable ' - '%d.' % (num_mutations, _MAX_MUTATIONS)) - request_pb = messages_pb2.MutateRowRequest( - table_name=self._table.name, - row_key=self._row_key, - mutations=mutations_list, - ) - # We expect a `google.protobuf.empty_pb2.Empty` - client = self._table._cluster._client - client._data_stub.MutateRow(request_pb, client.timeout_seconds) - - def _commit_check_and_mutate(self): - """Makes a ``CheckAndMutateRow`` API request. - - Assumes a filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. - - :rtype: bool - :returns: Flag indicating if the filter was matched (which also - indicates which set of mutations were applied by the server). - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - true_mutations = self._get_mutations(state=True) - false_mutations = self._get_mutations(state=False) - num_true_mutations = len(true_mutations) - num_false_mutations = len(false_mutations) - if num_true_mutations == 0 and num_false_mutations == 0: - return - if (num_true_mutations > _MAX_MUTATIONS or - num_false_mutations > _MAX_MUTATIONS): - raise ValueError( - 'Exceed the maximum allowable mutations (%d). Had %s true ' - 'mutations and %d false mutations.' % ( - _MAX_MUTATIONS, num_true_mutations, num_false_mutations)) - - request_pb = messages_pb2.CheckAndMutateRowRequest( - table_name=self._table.name, - row_key=self._row_key, - predicate_filter=self._filter.to_pb(), - true_mutations=true_mutations, - false_mutations=false_mutations, - ) - # We expect a `.messages_pb2.CheckAndMutateRowResponse` - client = self._table._cluster._client - resp = client._data_stub.CheckAndMutateRow( - request_pb, client.timeout_seconds) - return resp.predicate_matched - - def clear_mutations(self): - """Removes all currently accumulated mutations on the current row.""" - if self._filter is None: - del self._pb_mutations[:] - else: - del self._true_pb_mutations[:] - del self._false_pb_mutations[:] - - def commit(self): - """Makes a ``MutateRow`` or ``CheckAndMutateRow`` API request. - - If no mutations have been created in the row, no request is made. - - Mutations are applied atomically and in order, meaning that earlier - mutations can be masked / negated by later ones. Cells already present - in the row are left unchanged unless explicitly changed by a mutation. - - After committing the accumulated mutations, resets the local - mutations to an empty list. - - In the case that a filter is set on the :class:`Row`, the mutations - will be applied conditionally, based on whether the filter matches - any cells in the :class:`Row` or not. (Each method which adds a - mutation has a ``state`` parameter for this purpose.) - - :rtype: :class:`bool` or :data:`NoneType ` - :returns: :data:`None` if there is no filter, otherwise a flag - indicating if the filter was matched (which also - indicates which set of mutations were applied by the server). - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - if self._filter is None: - result = self._commit_mutate() - else: - result = self._commit_check_and_mutate() - - # Reset mutations after commit-ing request. - self.clear_mutations() - - return result - - def clear_modification_rules(self): - """Removes all currently accumulated modifications on current row.""" - del self._rule_pb_list[:] - - def commit_modifications(self): - """Makes a ``ReadModifyWriteRow`` API request. - - This commits modifications made by :meth:`append_cell_value` and - :meth:`increment_cell_value`. If no modifications were made, makes - no API request and just returns ``{}``. - - Modifies a row atomically, reading the latest existing timestamp/value - from the specified columns and writing a new value by appending / - incrementing. The new cell created uses either the current server time - or the highest timestamp of a cell in that column (if it exceeds the - server time). - - :rtype: dict - :returns: The new contents of all modified cells. Returned as a - dictionary of column families, each of which holds a - dictionary of columns. Each column contains a list of cells - modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } - """ - if len(self._rule_pb_list) == 0: - return {} - request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=self._table.name, - row_key=self._row_key, - rules=self._rule_pb_list, - ) - # We expect a `.data_pb2.Row` - client = self._table._cluster._client - row_response = client._data_stub.ReadModifyWriteRow( - request_pb, client.timeout_seconds) - - # Reset modifications after commit-ing request. - self.clear_modification_rules() - - # NOTE: We expect row_response.key == self._row_key but don't check. - return _parse_rmw_row_response(row_response) - - -class RowFilter(object): - """Basic filter to apply to cells in a row. - - These values can be combined via :class:`RowFilterChain`, - :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - - .. note:: - - This class is a do-nothing base class for all row filters. - """ - - def __ne__(self, other): - return not self.__eq__(other) - - -class _BoolFilter(RowFilter): - """Row filter that uses a boolean flag. - - :type flag: bool - :param flag: An indicator if a setting is turned on or off. - """ - - def __init__(self, flag): - self.flag = flag - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.flag == self.flag - - -class SinkFilter(_BoolFilter): - """Advanced row filter to skip parent filters. - - :type flag: bool - :param flag: ADVANCED USE ONLY. Hook for introspection into the row filter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(sink=self.flag) - - -class PassAllFilter(_BoolFilter): - """Row filter equivalent to not filtering at all. - - :type flag: bool - :param flag: Matches all cells, regardless of input. Functionally - equivalent to leaving ``filter`` unset, but included for - completeness. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(pass_all_filter=self.flag) - - -class BlockAllFilter(_BoolFilter): - """Row filter that doesn't match any cells. - - :type flag: bool - :param flag: Does not match any cells, regardless of input. Useful for - temporarily disabling just part of a filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(block_all_filter=self.flag) - - -class _RegexFilter(RowFilter): - """Row filter that uses a regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: bytes or str - :param regex: A regular expression (RE2) for some row filter. - """ - - def __init__(self, regex): - self.regex = regex - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.regex == self.regex - - -class RowKeyRegexFilter(_RegexFilter): - """Row filter for a row key regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from rows with row - keys that satisfy this regex. For a - ``CheckAndMutateRowRequest``, this filter is unnecessary - since the row key is already specified. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_key_regex_filter=self.regex) - - -class RowSampleFilter(RowFilter): - """Matches all cells from a row with probability p. - - :type sample: float - :param sample: The probability of matching a cell (must be in the - interval ``[0, 1]``). - """ - - def __init__(self, sample): - self.sample = sample - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.sample == self.sample - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(row_sample_filter=self.sample) - - -class FamilyNameRegexFilter(_RegexFilter): - """Row filter for a family name regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - :type regex: str - :param regex: A regular expression (RE2) to match cells from columns in a - given column family. For technical reasons, the regex must - not contain the ``':'`` character, even if it is not being - used as a literal. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(family_name_regex_filter=self.regex) - - -class ColumnQualifierRegexFilter(_RegexFilter): - """Row filter for a column qualifier regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells from column that - match this regex (irrespective of column family). - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(column_qualifier_regex_filter=self.regex) - - -class TimestampRange(object): - """Range of time with inclusive lower and exclusive upper bounds. - - :type start: :class:`datetime.datetime` - :param start: (Optional) The (inclusive) lower bound of the timestamp - range. If omitted, defaults to Unix epoch. - - :type end: :class:`datetime.datetime` - :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, no upper bound is used. - """ - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start == self.start and - other.end == self.end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`TimestampRange` to a protobuf. - - :rtype: :class:`.data_pb2.TimestampRange` - :returns: The converted current object. - """ - timestamp_range_kwargs = {} - if self.start is not None: - timestamp_range_kwargs['start_timestamp_micros'] = ( - _microseconds_from_datetime(self.start)) - if self.end is not None: - timestamp_range_kwargs['end_timestamp_micros'] = ( - _microseconds_from_datetime(self.end)) - return data_pb2.TimestampRange(**timestamp_range_kwargs) - - -class TimestampRangeFilter(RowFilter): - """Row filter that limits cells to a range of time. - - :type range_: :class:`TimestampRange` - :param range_: Range of time that cells should match against. - """ - - def __init__(self, range_): - self.range_ = range_ - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.range_ == self.range_ - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts the ``range_`` on the current object to a protobuf and - then uses it in the ``timestamp_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(timestamp_range_filter=self.range_.to_pb()) - - -class ColumnRangeFilter(RowFilter): - """A row filter to restrict to a range of columns. - - Both the start and end column can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must - be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is - used, the backend applies no upper bound to the - values. - - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_column`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_column`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_column`` is given or if ``inclusive_end`` - is set but no ``end_column`` is given - """ - - def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=None, inclusive_end=None): - self.column_family_id = column_family_id - - if inclusive_start is None: - inclusive_start = True - elif start_column is None: - raise ValueError('Inclusive start was specified but no ' - 'start column was given.') - self.start_column = start_column - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_column is None: - raise ValueError('Inclusive end was specified but no ' - 'end column was given.') - self.end_column = end_column - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.start_column == self.start_column and - other.end_column == self.end_column and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ColumnRange` and then uses it - in the ``column_range_filter`` field. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - column_range_kwargs = {'family_name': self.column_family_id} - if self.start_column is not None: - if self.inclusive_start: - key = 'start_qualifier_inclusive' - else: - key = 'start_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: - if self.inclusive_end: - key = 'end_qualifier_inclusive' - else: - key = 'end_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.end_column) - - column_range = data_pb2.ColumnRange(**column_range_kwargs) - return data_pb2.RowFilter(column_range_filter=column_range) - - -class ValueRegexFilter(_RegexFilter): - """Row filter for a value regular expression. - - The ``regex`` must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - Special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type regex: bytes - :param regex: A regular expression (RE2) to match cells with values that - match this regex. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(value_regex_filter=self.regex) - - -class ValueRangeFilter(RowFilter): - """A range of values to restrict to in a row filter. - - Will only match cells that have values in this range. - - Both the start and end value can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type start_value: bytes - :param start_value: The start of the range of values. If no value is used, - the backend applies no lower bound to the values. - - :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, - the backend applies no upper bound to the values. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). Defaults - to :data:`True` if ``start_value`` is passed and - no ``inclusive_start`` was given. - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). Defaults - to :data:`True` if ``end_value`` is passed and - no ``inclusive_end`` was given. - - :raises: :class:`ValueError ` if ``inclusive_start`` - is set but no ``start_value`` is given or if ``inclusive_end`` - is set but no ``end_value`` is given - """ - - def __init__(self, start_value=None, end_value=None, - inclusive_start=None, inclusive_end=None): - if inclusive_start is None: - inclusive_start = True - elif start_value is None: - raise ValueError('Inclusive start was specified but no ' - 'start value was given.') - self.start_value = start_value - self.inclusive_start = inclusive_start - - if inclusive_end is None: - inclusive_end = True - elif end_value is None: - raise ValueError('Inclusive end was specified but no ' - 'end value was given.') - self.end_value = end_value - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start_value == self.start_value and - other.end_value == self.end_value and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def to_pb(self): - """Converts the row filter to a protobuf. - - First converts to a :class:`.data_pb2.ValueRange` and then uses - it to create a row filter protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - value_range_kwargs = {} - if self.start_value is not None: - if self.inclusive_start: - key = 'start_value_inclusive' - else: - key = 'start_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.start_value) - if self.end_value is not None: - if self.inclusive_end: - key = 'end_value_inclusive' - else: - key = 'end_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.end_value) - - value_range = data_pb2.ValueRange(**value_range_kwargs) - return data_pb2.RowFilter(value_range_filter=value_range) - - -class _CellCountFilter(RowFilter): - """Row filter that uses an integer count of cells. - - The cell count is used as an offset or a limit for the number - of results returned. - - :type num_cells: int - :param num_cells: An integer count / offset / limit. - """ - - def __init__(self, num_cells): - self.num_cells = num_cells - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.num_cells == self.num_cells - - -class CellsRowOffsetFilter(_CellCountFilter): - """Row filter to skip cells in a row. - - :type num_cells: int - :param num_cells: Skips the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_offset_filter=self.num_cells) - - -class CellsRowLimitFilter(_CellCountFilter): - """Row filter to limit cells in a row. - - :type num_cells: int - :param num_cells: Matches only the first N cells of the row. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_row_limit_filter=self.num_cells) - - -class CellsColumnLimitFilter(_CellCountFilter): - """Row filter to limit cells in a column. - - :type num_cells: int - :param num_cells: Matches only the most recent N cells within each column. - This filters a (family name, column) pair, based on - timestamps of each cell. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(cells_per_column_limit_filter=self.num_cells) - - -class StripValueTransformerFilter(_BoolFilter): - """Row filter that transforms cells into empty string (0 bytes). - - :type flag: bool - :param flag: If :data:`True`, replaces each cell's value with the empty - string. As the name indicates, this is more useful as a - transformer than a generic query / filter. - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(strip_value_transformer=self.flag) - - -class ApplyLabelFilter(RowFilter): - """Filter to apply labels to cells. - - Intended to be used as an intermediate filter on a pre-existing filtered - result set. This was if two sets are combined, the label can tell where - the cell(s) originated.This allows the client to determine which results - were produced from which part of the filter. - - .. note:: - - Due to a technical limitation, it is not currently possible to apply - multiple labels to a cell. - - :type label: str - :param label: Label to apply to cells in the output row. Values must be - at most 15 characters long, and match the pattern - ``[a-z0-9\\-]+``. - """ - - def __init__(self, label): - self.label = label - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.label == self.label - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - return data_pb2.RowFilter(apply_label_transformer=self.label) - - -class _FilterCombination(RowFilter): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def __init__(self, filters=None): - if filters is None: - filters = [] - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - -class RowFilterChain(_FilterCombination): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - chain = data_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(chain=chain) - - -class RowFilterUnion(_FilterCombination): - """Union of row filters. - - Sends rows through several filters simultaneously, then - merges / interleaves all the filtered results together. - - If multiple cells are produced with the same column and timestamp, - they will all appear in the output row in an unspecified mutual order. - - :type filters: list - :param filters: List of :class:`RowFilter` - """ - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - interleave = data_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(interleave=interleave) - - -class ConditionalRowFilter(RowFilter): - """Conditional row filter which exhibits ternary behavior. - - Executes one of two filters based on another filter. If the ``base_filter`` - returns any cells in the row, then ``true_filter`` is executed. If not, - then ``false_filter`` is executed. - - .. note:: - - The ``base_filter`` does not execute atomically with the true and false - filters, which may lead to inconsistent or unexpected results. - - Additionally, executing a :class:`ConditionalRowFilter` has poor - performance on the server, especially when ``false_filter`` is set. - - :type base_filter: :class:`RowFilter` - :param base_filter: The filter to condition on before executing the - true/false filters. - - :type true_filter: :class:`RowFilter` - :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results - will be returned in the true case. - - :type false_filter: :class:`RowFilter` - :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results - will be returned in the false case. - """ - - def __init__(self, base_filter, true_filter=None, false_filter=None): - self.base_filter = base_filter - self.true_filter = true_filter - self.false_filter = false_filter - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.base_filter == self.base_filter and - other.true_filter == self.true_filter and - other.false_filter == self.false_filter) - - def to_pb(self): - """Converts the row filter to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} - if self.true_filter is not None: - condition_kwargs['true_filter'] = self.true_filter.to_pb() - if self.false_filter is not None: - condition_kwargs['false_filter'] = self.false_filter.to_pb() - condition = data_pb2.RowFilter.Condition(**condition_kwargs) - return data_pb2.RowFilter(condition=condition) - - -def _parse_rmw_row_response(row_response): - """Parses the response to a ``ReadModifyWriteRow`` request. - - :type row_response: :class:`.data_pb2.Row` - :param row_response: The response row (with only modified cells) from a - ``ReadModifyWriteRow`` request. - - :rtype: dict - :returns: The new contents of all modified cells. Returned as a - dictionary of column families, each of which holds a - dictionary of columns. Each column contains a list of cells - modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } - """ - result = {} - for column_family in row_response.families: - column_family_id, curr_family = _parse_family_pb(column_family) - result[column_family_id] = curr_family - return result - - -def _parse_family_pb(family_pb): - """Parses a Family protobuf into a dictionary. - - :type family_pb: :class:`._generated.bigtable_data_pb2.Family` - :param family_pb: A protobuf - - :rtype: tuple - :returns: A string and dictionary. The string is the name of the - column family and the dictionary has column names (within the - family) as keys and cell lists as values. Each cell is - represented with a two-tuple with the value (in bytes) and the - timestamp for the cell. For example: - - .. code:: python - - { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - } - """ - result = {} - for column in family_pb.columns: - result[column.qualifier] = cells = [] - for cell in column.cells: - val_pair = ( - cell.value, - _datetime_from_microseconds(cell.timestamp_micros), - ) - cells.append(val_pair) - - return family_pb.name, result diff --git a/gcloud_bigtable/row_data.py b/gcloud_bigtable/row_data.py deleted file mode 100644 index e0ec054..0000000 --- a/gcloud_bigtable/row_data.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Container for Google Cloud Bigtable Cells and Streaming Row Contents.""" - - -import copy -import six - -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -from gcloud_bigtable._non_upstream_helpers import _to_bytes - -# pylint: disable=invalid-name -# Alias for easy upstream diffs. -_datetime_from_microseconds = _microseconds_to_timestamp -# pylint: enable=invalid-name - - -class Cell(object): - """Representation of a Google Cloud Bigtable Cell. - - :type value: bytes - :param value: The value stored in the cell. - - :type timestamp: :class:`datetime.datetime` - :param timestamp: The timestamp when the cell was stored. - - :type labels: list - :param labels: (Optional) List of strings. Labels applied to the cell. - """ - - def __init__(self, value, timestamp, labels=()): - self.value = value - self.timestamp = timestamp - self.labels = list(labels) - - @classmethod - def from_pb(cls, cell_pb): - """Create a new cell from a Cell protobuf. - - :type cell_pb: :class:`._generated.bigtable_data_pb2.Cell` - :param cell_pb: The protobuf to convert. - - :rtype: :class:`Cell` - :returns: The cell corresponding to the protobuf. - """ - timestamp = _datetime_from_microseconds(cell_pb.timestamp_micros) - if cell_pb.labels: - return cls(cell_pb.value, timestamp, labels=cell_pb.labels) - else: - return cls(cell_pb.value, timestamp) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.value == self.value and - other.timestamp == self.timestamp and - other.labels == self.labels) - - def __ne__(self, other): - return not self.__eq__(other) - - -class PartialRowData(object): - """Representation of partial row in a Google Cloud Bigtable Table. - - These are expected to be updated directly from a - :class:`._generated.bigtable_service_messages_pb2.ReadRowsResponse` - - :type row_key: bytes - :param row_key: The key for the row holding the (partial) data. - """ - - def __init__(self, row_key): - self._row_key = row_key - self._cells = {} - self._committed = False - self._chunks_encountered = False - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other._row_key == self._row_key and - other._committed == self._committed and - other._chunks_encountered == self._chunks_encountered and - other._cells == self._cells) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_dict(self): - """Convert the cells to a dictionary. - - This is intended to be used with HappyBase, so the column family and - column qualiers are combined (with ``:``). - - :rtype: dict - :returns: Dictionary containing all the data in the cells of this row. - """ - result = {} - for column_family_id, columns in six.iteritems(self._cells): - for column_qual, cells in six.iteritems(columns): - key = (_to_bytes(column_family_id) + b':' + - _to_bytes(column_qual)) - result[key] = cells - return result - - @property - def cells(self): - """Property returning all the cells accumulated on this partial row. - - :rtype: dict - :returns: Dictionary of the :class:`Cell` objects accumulated. This - dictionary has two-levels of keys (first for column families - and second for column names/qualifiers within a family). For - a given column, a list of :class:`Cell` objects is stored. - """ - return copy.deepcopy(self._cells) - - @property - def row_key(self): - """Getter for the current (partial) row's key. - - :rtype: bytes - :returns: The current (partial) row's key. - """ - return self._row_key - - @property - def committed(self): - """Getter for the committed status of the (partial) row. - - :rtype: bool - :returns: The committed status of the (partial) row. - """ - return self._committed - - def clear(self): - """Clears all cells that have been added.""" - self._committed = False - self._chunks_encountered = False - self._cells.clear() - - def _handle_commit_row(self, chunk, index, last_chunk_index): - """Handles a ``commit_row`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - - :type index: int - :param index: The current index of the chunk. - - :type last_chunk_index: int - :param last_chunk_index: The index of the last chunk. - - :raises: :class:`ValueError ` if the value of - ``commit_row`` is :data:`False` or if the chunk passed is not - the last chunk in a response. - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``commit_row``. - if not chunk.commit_row: - raise ValueError('Received commit_row that was False.') - - if index != last_chunk_index: - raise ValueError('Commit row chunk was not the last chunk') - else: - self._committed = True - - def _handle_reset_row(self, chunk): - """Handles a ``reset_row`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - - :raises: :class:`ValueError ` if the value of - ``reset_row`` is :data:`False` - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``reset_row``. - if not chunk.reset_row: - raise ValueError('Received reset_row that was False.') - - self.clear() - - def _handle_row_contents(self, chunk): - """Handles a ``row_contents`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``row_contents``. - - # chunk.row_contents is ._generated.bigtable_data_pb2.Family - column_family_id = chunk.row_contents.name - column_family_dict = self._cells.setdefault(column_family_id, {}) - for column in chunk.row_contents.columns: - cells = [Cell.from_pb(cell) for cell in column.cells] - - column_name = column.qualifier - column_cells = column_family_dict.setdefault(column_name, []) - column_cells.extend(cells) - - def update_from_read_rows(self, read_rows_response_pb): - """Updates the current row from a ``ReadRows`` response. - - :type read_rows_response_pb: - :class:`._generated.bigtable_service_messages_pb2.ReadRowsResponse` - :param read_rows_response_pb: A response streamed back as part of a - ``ReadRows`` request. - - :raises: :class:`ValueError ` if the current - partial row has already been committed, if the row key on the - response doesn't match the current one or if there is a chunk - encountered with an unexpected ``ONEOF`` protobuf property. - """ - if self._committed: - raise ValueError('The row has been committed') - - if read_rows_response_pb.row_key != self.row_key: - raise ValueError('Response row key (%r) does not match current ' - 'one (%r).' % (read_rows_response_pb.row_key, - self.row_key)) - - last_chunk_index = len(read_rows_response_pb.chunks) - 1 - for index, chunk in enumerate(read_rows_response_pb.chunks): - chunk_property = chunk.WhichOneof('chunk') - if chunk_property == 'row_contents': - self._handle_row_contents(chunk) - elif chunk_property == 'reset_row': - self._handle_reset_row(chunk) - elif chunk_property == 'commit_row': - self._handle_commit_row(chunk, index, last_chunk_index) - else: - # NOTE: This includes chunk_property == None since we always - # want a value to be set - raise ValueError('Unexpected chunk property: %s' % ( - chunk_property,)) - - self._chunks_encountered = True - - -class PartialRowsData(object): - """Convenience wrapper for consuming a ``ReadRows`` streaming response. - - :type response_iterator: - :class:`grpc.framework.alpha._reexport._CancellableIterator` - :param response_iterator: A streaming iterator returned from a - ``ReadRows`` request. - """ - - def __init__(self, response_iterator): - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - self._response_iterator = response_iterator - self._rows = {} - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other._response_iterator == self._response_iterator - - def __ne__(self, other): - return not self.__eq__(other) - - @property - def rows(self): - """Property returning all rows accumulated from the stream. - - :rtype: dict - :returns: Dictionary of :class:`PartialRowData`. - """ - # NOTE: To avoid duplicating large objects, this is just the - # mutable private data. - return self._rows - - def cancel(self): - """Cancels the iterator, closing the stream.""" - self._response_iterator.cancel() - - def consume_next(self): - """Consumes the next ``ReadRowsResponse`` from the stream. - - Parses the response and stores it as a :class:`PartialRowData` - in a dictionary owned by this object. - - :raises: :class:`StopIteration ` if the - response iterator has no more responses to stream. - """ - read_rows_response = self._response_iterator.next() - row_key = read_rows_response.row_key - partial_row = self._rows.get(row_key) - if partial_row is None: - partial_row = self._rows[row_key] = PartialRowData(row_key) - # NOTE: This is not atomic in the case of failures. - partial_row.update_from_read_rows(read_rows_response) - - def consume_all(self, max_loops=None): - """Consume the streamed responses until there are no more. - - This simply calls :meth:`consume_next` until there are no - more to consume. - - :type max_loops: int - :param max_loops: (Optional) Maximum number of times to try to consume - an additional ``ReadRowsResponse``. You can use this - to avoid long wait times. - """ - curr_loop = 0 - if max_loops is None: - max_loops = float('inf') - while curr_loop < max_loops: - curr_loop += 1 - try: - self.consume_next() - except StopIteration: - break diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py deleted file mode 100644 index 28386ff..0000000 --- a/gcloud_bigtable/table.py +++ /dev/null @@ -1,423 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Table.""" - - -from gcloud_bigtable._non_upstream_helpers import _to_bytes -from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as data_messages_pb2) -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable.column_family import _gc_rule_from_pb -from gcloud_bigtable.column_family import ColumnFamily -from gcloud_bigtable.row import Row -from gcloud_bigtable.row_data import PartialRowData -from gcloud_bigtable.row_data import PartialRowsData - - -class Table(object): - """Representation of a Google Cloud Bigtable Table. - - .. note:: - - We don't define any properties on a table other than the name. As - the proto says, in a request: - - The ``name`` field of the Table and all of its ColumnFamilies must - be left blank, and will be populated in the response. - - This leaves only the ``current_operation`` and ``granularity`` - fields. The ``current_operation`` is only used for responses while - ``granularity`` is an enum with only one value. - - We can use a :class:`Table` to: - - * :meth:`create` the table - * :meth:`rename` the table - * :meth:`delete` the table - * :meth:`list_column_families` in the table - - :type table_id: str - :param table_id: The ID of the table. - - :type cluster: :class:`.cluster.Cluster` - :param cluster: The cluster that owns the table. - """ - - def __init__(self, table_id, cluster): - self.table_id = table_id - self._cluster = cluster - - @property - def name(self): - """Table name used in requests. - - .. note:: - - This property will not change if ``table_id`` does not, but the - return value is not cached. - - The table name is of the form - - ``"projects/../zones/../clusters/../tables/{table_id}"`` - - :rtype: str - :returns: The table name. - """ - return self._cluster.name + '/tables/' + self.table_id - - def column_family(self, column_family_id, gc_rule=None): - """Factory to create a column family associated with this table. - - :type column_family_id: str - :param column_family_id: The ID of the column family. Must be of the - form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type gc_rule: :class:`.column_family.GarbageCollectionRule` - :param gc_rule: (Optional) The garbage collection settings for this - column family. - - :rtype: :class:`.column_family.ColumnFamily` - :returns: A column family owned by this table. - """ - return ColumnFamily(column_family_id, self, gc_rule=gc_rule) - - def row(self, row_key, filter_=None): - """Factory to create a row associated with this table. - - :type row_key: bytes - :param row_key: The key for the row being created. - - :type filter_: :class:`.RowFilter` - :param filter_: (Optional) Filter to be used for conditional mutations. - See :class:`.Row` for more details. - - :rtype: :class:`.Row` - :returns: A row owned by this table. - """ - return Row(row_key, self, filter_=filter_) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.table_id == self.table_id and - other._cluster == self._cluster) - - def __ne__(self, other): - return not self.__eq__(other) - - def create(self, initial_split_keys=None): - """Creates this table. - - .. note:: - - Though a :class:`._generated.bigtable_table_data_pb2.Table` is also - allowed (as the ``table`` property) in a create table request, we - do not support it in this method. As mentioned in the - :class:`Table` docstring, the name is the only useful property in - the table proto. - - .. note:: - - A create request returns a - :class:`._generated.bigtable_table_data_pb2.Table` but we don't use - this response. The proto definition allows for the inclusion of a - ``current_operation`` in the response, but it does not appear that - the Cloud Bigtable API returns any operation. - - :type initial_split_keys: list - :param initial_split_keys: (Optional) List of row keys that will be - used to initially split the table into - several tablets (Tablets are similar to - HBase regions). Given two split keys, - ``"s1"`` and ``"s2"``, three tablets will be - created, spanning the key ranges: - ``[, s1)``, ``[s1, s2)``, ``[s2, )``. - """ - request_pb = messages_pb2.CreateTableRequest( - initial_split_keys=initial_split_keys or [], - name=self._cluster.name, - table_id=self.table_id, - ) - client = self._cluster._client - # We expect a `._generated.bigtable_table_data_pb2.Table` - client._table_stub.CreateTable(request_pb, client.timeout_seconds) - - def rename(self, new_table_id): - """Rename this table. - - .. note:: - - This cannot be used to move tables between clusters, - zones, or projects. - - .. note:: - - The Bigtable Table Admin API currently (``v1``) returns - - ``BigtableTableService.RenameTable is not yet implemented`` - - when this method is used. It's unclear when this method will - actually be supported by the API. - - :type new_table_id: str - :param new_table_id: The new name table ID. - """ - request_pb = messages_pb2.RenameTableRequest( - name=self.name, - new_id=new_table_id, - ) - client = self._cluster._client - # We expect a `google.protobuf.empty_pb2.Empty` - client._table_stub.RenameTable(request_pb, client.timeout_seconds) - - self.table_id = new_table_id - - def delete(self): - """Delete this table.""" - request_pb = messages_pb2.DeleteTableRequest(name=self.name) - client = self._cluster._client - # We expect a `google.protobuf.empty_pb2.Empty` - client._table_stub.DeleteTable(request_pb, client.timeout_seconds) - - def list_column_families(self): - """List the column families owned by this table. - - :rtype: dict - :returns: Dictionary of column families attached to this table. Keys - are strings (column family names) and values are - :class:`.column_family.ColumnFamily` instances. - :raises: :class:`ValueError ` if the column - family name from the response does not agree with the computed - name from the column family ID. - """ - request_pb = messages_pb2.GetTableRequest(name=self.name) - client = self._cluster._client - # We expect a `._generated.bigtable_table_data_pb2.Table` - table_pb = client._table_stub.GetTable(request_pb, - client.timeout_seconds) - - result = {} - for column_family_id, value_pb in table_pb.column_families.items(): - gc_rule = _gc_rule_from_pb(value_pb.gc_rule) - column_family = self.column_family(column_family_id, - gc_rule=gc_rule) - if column_family.name != value_pb.name: - raise ValueError('Column family name %s does not agree with ' - 'name from request: %s.' % ( - column_family.name, value_pb.name)) - result[column_family_id] = column_family - return result - - def read_row(self, row_key, filter_=None): - """Read a single row from this table. - - :type row_key: bytes - :param row_key: The key of the row to read from. - - :type filter_: :class:`.row.RowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - row. If unset, returns the entire row. - - :rtype: :class:`.PartialRowData`, :data:`NoneType ` - :returns: The contents of the row if any chunks were returned in - the response, otherwise :data:`None`. - :raises: :class:`ValueError ` if a commit row - chunk is never encountered. - """ - request_pb = _create_row_request(self.name, row_key=row_key, - filter_=filter_) - client = self._cluster._client - response_iterator = client._data_stub.ReadRows(request_pb, - client.timeout_seconds) - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - result = PartialRowData(row_key) - for read_rows_response in response_iterator: - result.update_from_read_rows(read_rows_response) - - # Make sure the result actually contains data. - if not result._chunks_encountered: - return None - # Make sure the result was committed by the back-end. - if not result.committed: - raise ValueError('The row remains partial / is not committed.') - return result - - def read_rows(self, start_key=None, end_key=None, - allow_row_interleaving=None, limit=None, filter_=None): - """Read rows from this table. - - :type start_key: bytes - :param start_key: (Optional) The beginning of a range of row keys to - read from. The range will include ``start_key``. If - left empty, will be interpreted as the empty string. - - :type end_key: bytes - :param end_key: (Optional) The end of a range of row keys to read from. - The range will not include ``end_key``. If left empty, - will be interpreted as an infinite string. - - :type allow_row_interleaving: bool - :param allow_row_interleaving: (Optional) By default, rows are read - sequentially, producing results which - are guaranteed to arrive in increasing - row order. Setting - ``allow_row_interleaving`` to - :data:`True` allows multiple rows to be - interleaved in the response stream, - which increases throughput but breaks - this guarantee, and may force the - client to use more memory to buffer - partially-received rows. - - :type limit: int - :param limit: (Optional) The read will terminate after committing to N - rows' worth of results. The default (zero) is to return - all results. Note that if ``allow_row_interleaving`` is - set to :data:`True`, partial results may be returned for - more than N rows. However, only N ``commit_row`` chunks - will be sent. - - :type filter_: :class:`.row.RowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - specified row(s). If unset, reads every column in - each row. - - :rtype: :class:`.PartialRowsData` - :returns: A :class:`.PartialRowsData` convenience wrapper for consuming - the streamed results. - """ - request_pb = _create_row_request( - self.name, start_key=start_key, end_key=end_key, filter_=filter_, - allow_row_interleaving=allow_row_interleaving, limit=limit) - client = self._cluster._client - response_iterator = client._data_stub.ReadRows(request_pb, - client.timeout_seconds) - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - return PartialRowsData(response_iterator) - - def sample_row_keys(self): - """Read a sample of row keys in the table. - - The returned row keys will delimit contiguous sections of the table of - approximately equal size, which can be used to break up the data for - distributed tasks like mapreduces. - - The elements in the iterator are a SampleRowKeys response and they have - the properties ``offset_bytes`` and ``row_key``. They occur in sorted - order. The table might have contents before the first row key in the - list and after the last one, but a key containing the empty string - indicates "end of table" and will be the last response given, if - present. - - .. note:: - - Row keys in this list may not have ever been written to or read - from, and users should therefore not make any assumptions about the - row key structure that are specific to their use case. - - The ``offset_bytes`` field on a response indicates the approximate - total storage space used by all rows in the table which precede - ``row_key``. Buffering the contents of all rows between two subsequent - samples would require space roughly equal to the difference in their - ``offset_bytes`` fields. - - :rtype: :class:`grpc.framework.alpha._reexport._CancellableIterator` - :returns: A cancel-able iterator. Can be consumed by calling ``next()`` - or by casting to a :class:`list` and can be cancelled by - calling ``cancel()``. - """ - request_pb = data_messages_pb2.SampleRowKeysRequest( - table_name=self.name) - client = self._cluster._client - response_iterator = client._data_stub.SampleRowKeys( - request_pb, client.timeout_seconds) - return response_iterator - - -def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, - filter_=None, allow_row_interleaving=None, limit=None): - """Creates a request to read rows in a table. - - :type table_name: str - :param table_name: The name of the table to read from. - - :type row_key: bytes - :param row_key: (Optional) The key of a specific row to read from. - - :type start_key: bytes - :param start_key: (Optional) The beginning of a range of row keys to - read from. The range will include ``start_key``. If - left empty, will be interpreted as the empty string. - - :type end_key: bytes - :param end_key: (Optional) The end of a range of row keys to read from. - The range will not include ``end_key``. If left empty, - will be interpreted as an infinite string. - - :type filter_: :class:`.row.RowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - specified row(s). If unset, reads the entire table. - - :type allow_row_interleaving: bool - :param allow_row_interleaving: (Optional) By default, rows are read - sequentially, producing results which are - guaranteed to arrive in increasing row - order. Setting - ``allow_row_interleaving`` to - :data:`True` allows multiple rows to be - interleaved in the response stream, - which increases throughput but breaks - this guarantee, and may force the - client to use more memory to buffer - partially-received rows. - - :type limit: int - :param limit: (Optional) The read will terminate after committing to N - rows' worth of results. The default (zero) is to return - all results. Note that if ``allow_row_interleaving`` is - set to :data:`True`, partial results may be returned for - more than N rows. However, only N ``commit_row`` chunks - will be sent. - - :rtype: :class:`data_messages_pb2.ReadRowsRequest` - :returns: The ``ReadRowsRequest`` protobuf corresponding to the inputs. - :raises: :class:`ValueError ` if both - ``row_key`` and one of ``start_key`` and ``end_key`` are set - """ - request_kwargs = {'table_name': table_name} - if (row_key is not None and - (start_key is not None or end_key is not None)): - raise ValueError('Row key and row range cannot be ' - 'set simultaneously') - if row_key is not None: - request_kwargs['row_key'] = _to_bytes(row_key) - if start_key is not None or end_key is not None: - range_kwargs = {} - if start_key is not None: - range_kwargs['start_key'] = _to_bytes(start_key) - if end_key is not None: - range_kwargs['end_key'] = _to_bytes(end_key) - row_range = data_pb2.RowRange(**range_kwargs) - request_kwargs['row_range'] = row_range - if filter_ is not None: - request_kwargs['filter'] = filter_.to_pb() - if allow_row_interleaving is not None: - request_kwargs['allow_row_interleaving'] = allow_row_interleaving - if limit is not None: - request_kwargs['num_rows_limit'] = limit - - return data_messages_pb2.ReadRowsRequest(**request_kwargs) diff --git a/gcloud_bigtable/test__non_upstream_helpers.py b/gcloud_bigtable/test__non_upstream_helpers.py deleted file mode 100644 index c619f3c..0000000 --- a/gcloud_bigtable/test__non_upstream_helpers.py +++ /dev/null @@ -1,416 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT = 'project-id' - - -class Test__project_from_environment(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._non_upstream_helpers import ( - _project_from_environment) - return _project_from_environment() - - def test_it(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - fake_project = object() - mock_os = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, os=mock_os): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_os._called, - [('getenv', (MUT.PROJECT_ENV_VAR,), {})]) - - -class Test__project_from_app_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._non_upstream_helpers import ( - _project_from_app_engine) - return _project_from_app_engine() - - def test_without_app_engine(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - with _Monkey(MUT, app_identity=None): - result = self._callFUT() - - self.assertEqual(result, None) - - def test_with_app_engine(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - fake_project = object() - mock_app_identity = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, app_identity=mock_app_identity): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_app_identity._called, - [('get_application_id', (), {})]) - - -class Test__project_from_compute_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._non_upstream_helpers import ( - _project_from_compute_engine) - return _project_from_compute_engine() - - @staticmethod - def _make_http_connection_response(status_code, read_result, - raise_socket_err=False): - import socket - - class Response(object): - status = status_code - - @staticmethod - def read(): - if raise_socket_err: - raise socket.error('Failed') - else: - return read_result - return Response - - @staticmethod - def _make_fake_six_module(mock_http_client): - class MockSix(object): - class moves(object): - http_client = mock_http_client - return MockSix - - def _helper(self, status, raise_socket_err=False): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - fake_project = object() - response = self._make_http_connection_response( - status, fake_project, raise_socket_err=raise_socket_err) - # The connection does the bulk of the work. - mock_connection = _MockWithAttachedMethods(None, response, None) - # The http_client module holds the connection constructor. - mock_http_client = _MockWithAttachedMethods(mock_connection) - # We need to put the client in place of it's location in six. - mock_six = self._make_fake_six_module(mock_http_client) - - with _Monkey(MUT, six=mock_six): - result = self._callFUT() - - if status == 200 and not raise_socket_err: - self.assertEqual(result, fake_project) - else: - self.assertEqual(result, None) - - self.assertEqual(mock_connection._called, [ - ( - 'request', - ('GET', '/computeMetadata/v1/project/project-id'), - {'headers': {'Metadata-Flavor': 'Google'}}, - ), - ( - 'getresponse', - (), - {}, - ), - ( - 'close', - (), - {}, - ), - ]) - self.assertEqual(mock_http_client._called, [ - ( - 'HTTPConnection', - ('169.254.169.254',), - {'timeout': 0.1}, - ), - ]) - - def test_success(self): - self._helper(200) - - def test_failed_status(self): - self._helper(404) - - def test_read_fails_with_socket_error(self): - self._helper(200, raise_socket_err=True) - - -class Test__determine_project(unittest2.TestCase): - - def _callFUT(self, project): - from gcloud_bigtable._non_upstream_helpers import _determine_project - return _determine_project(project) - - def _helper(self, num_mocks_called, mock_output, method_input): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - mock_project_from_environment = _MockCalled(None) - mock_project_from_app_engine = _MockCalled(None) - mock_project_from_compute_engine = _MockCalled(None) - - monkey_kwargs = { - '_project_from_environment': mock_project_from_environment, - '_project_from_app_engine': mock_project_from_app_engine, - '_project_from_compute_engine': ( - mock_project_from_compute_engine), - } - # Need the mocks in order they are called, so we can - # access them based on `num_mocks_called`. - mocks = [ - mock_project_from_environment, - mock_project_from_app_engine, - mock_project_from_compute_engine, - ] - mocks[num_mocks_called - 1].result = mock_output - - with _Monkey(MUT, **monkey_kwargs): - if num_mocks_called == 3 and mock_output is None: - with self.assertRaises(EnvironmentError): - self._callFUT(method_input) - else: - result = self._callFUT(method_input) - self.assertEqual(result, method_input or mock_output) - - # Make sure our mocks were called with no arguments. - for mock in mocks[:num_mocks_called]: - mock.check_called(self, [()]) - for mock in mocks[num_mocks_called:]: - mock.check_called(self, []) - - def test_fail_to_infer(self): - self._helper(num_mocks_called=3, mock_output=None, - method_input=None) - - def test_with_explicit_value(self): - self._helper(num_mocks_called=0, mock_output=None, - method_input=PROJECT) - - def test_from_environment(self): - self._helper(num_mocks_called=1, mock_output=PROJECT, - method_input=None) - - def test_from_app_engine(self): - self._helper(num_mocks_called=2, mock_output=PROJECT, - method_input=None) - - def test_from_compute_engine(self): - self._helper(num_mocks_called=3, mock_output=PROJECT, - method_input=None) - - -class Test__get_contents(unittest2.TestCase): - - def _callFUT(self, filename): - from gcloud_bigtable._non_upstream_helpers import _get_contents - return _get_contents(filename) - - def test_it(self): - import tempfile - - filename = tempfile.mktemp() - contents = b'foobar' - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - self.assertEqual(self._callFUT(filename), contents) - - -class Test__FactoryMixin(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable._non_upstream_helpers import _FactoryMixin - return _FactoryMixin - - def test_from_service_account_json(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - get_adc = _MockCalled(credentials) - json_credentials_path = 'JSON_CREDENTIALS_PATH' - - with _Monkey(MUT, - _get_application_default_credential_from_file=get_adc): - client = klass.from_service_account_json( - json_credentials_path, project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is credentials) - - self.assertEqual(credentials._called, []) - # _get_application_default_credential_from_file only has pos. args. - get_adc.check_called(self, [(json_credentials_path,)]) - - def test_from_service_account_p12(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _non_upstream_helpers as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - signed_creds = _MockServiceAccountCredentials(credentials) - - client_email = 'CLIENT_EMAIL' - private_key_path = 'PRIVATE_KEY_PATH' - - with _Monkey(MUT, ServiceAccountCredentials=signed_creds): - client = klass.from_service_account_p12( - client_email, private_key_path, project=PROJECT) - - self.assertEqual(signed_creds.p12_called, - [(client_email, private_key_path)]) - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is credentials) - self.assertEqual(credentials._called, []) - - -class Test__timestamp_to_microseconds(unittest2.TestCase): - - def _callFUT(self, timestamp, granularity=1000): - from gcloud_bigtable._non_upstream_helpers import ( - _timestamp_to_microseconds) - return _timestamp_to_microseconds(timestamp, granularity=granularity) - - def test_default_granularity(self): - import datetime - from gcloud_bigtable import _non_upstream_helpers as MUT - - microseconds = 898294371 - millis_granularity = microseconds - (microseconds % 1000) - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(millis_granularity, self._callFUT(timestamp)) - - def test_no_granularity(self): - import datetime - from gcloud_bigtable import _non_upstream_helpers as MUT - - microseconds = 11122205067 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(microseconds, self._callFUT(timestamp, granularity=1)) - - def test_non_utc_timestamp(self): - import datetime - - epoch_no_tz = datetime.datetime.utcfromtimestamp(0) - with self.assertRaises(TypeError): - self._callFUT(epoch_no_tz) - - def test_non_datetime_timestamp(self): - timestamp = object() # Not a datetime object. - with self.assertRaises(TypeError): - self._callFUT(timestamp) - - -class Test__microseconds_to_timestamp(unittest2.TestCase): - - def _callFUT(self, microseconds): - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - return _microseconds_to_timestamp(microseconds) - - def test_it(self): - import datetime - from gcloud_bigtable import _non_upstream_helpers as MUT - - microseconds = 123456 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(timestamp, self._callFUT(microseconds)) - - -class Test__to_bytes(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable._non_upstream_helpers import _to_bytes - return _to_bytes(*args, **kwargs) - - def test_with_bytes(self): - value = b'bytes-val' - self.assertEqual(self._callFUT(value), value) - - def test_with_unicode(self): - value = u'string-val' - encoded_value = b'string-val' - self.assertEqual(self._callFUT(value), encoded_value) - - def test_unicode_non_ascii(self): - value = u'\u2013' # Long hyphen - encoded_value = b'\xe2\x80\x93' - self.assertRaises(UnicodeEncodeError, self._callFUT, value) - self.assertEqual(self._callFUT(value, encoding='utf-8'), - encoded_value) - - def test_with_nonstring_type(self): - value = object() - self.assertRaises(TypeError, self._callFUT, value) - - -class Test__total_seconds_backport(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable._non_upstream_helpers import ( - _total_seconds_backport) - return _total_seconds_backport(*args, **kwargs) - - def test_it(self): - import datetime - offset = datetime.timedelta(seconds=3, - microseconds=140000) - result = self._callFUT(offset) - self.assertEqual(result, 3.14) - - -class Test__total_seconds(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable._non_upstream_helpers import _total_seconds - return _total_seconds(*args, **kwargs) - - def test_it(self): - import datetime - offset = datetime.timedelta(seconds=1, - microseconds=414000) - result = self._callFUT(offset) - self.assertEqual(result, 1.414) - - -class _MockServiceAccountCredentials(object): - - def __init__(self, result): - self.result = result - self.p12_called = [] - - def from_p12_keyfile(self, client_email, path): - self.p12_called.append((client_email, path)) - return self.result diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py deleted file mode 100644 index c1bbf30..0000000 --- a/gcloud_bigtable/test_client.py +++ /dev/null @@ -1,764 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT = 'project-id' - - -class TestClient(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.client import Client - return Client - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _constructor_test_helper(self, expected_scopes, project=None, - read_only=False, admin=False, - user_agent=None): - from gcloud_bigtable import _non_upstream_helpers as OTHER_MOD - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - determined_project = object() - mock_determine_project = _MockCalled(determined_project) - with _Monkey(OTHER_MOD, _determine_project=mock_determine_project): - client = self._makeOne(project=project, credentials=credentials, - read_only=read_only, admin=admin, - user_agent=user_agent) - - self.assertTrue(client._credentials is scoped_creds) - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - self.assertTrue(client.project is determined_project) - self.assertEqual(client.timeout_seconds, MUT.DEFAULT_TIMEOUT_SECONDS) - self.assertEqual(client.user_agent, user_agent) - mock_determine_project.check_called(self, [(project,)]) - - def test_constructor_default(self): - from gcloud_bigtable import _non_upstream_helpers as OTHER_MOD - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - mock_creds_class = _MockWithAttachedMethods(credentials) - - with _Monkey(OTHER_MOD, GoogleCredentials=mock_creds_class): - client = self._makeOne(project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - self.assertEqual(client.user_agent, MUT.DEFAULT_USER_AGENT) - self.assertEqual(mock_creds_class._called, - [('get_application_default', (), {})]) - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - - def test_constructor_explicit_credentials(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes) - - def test_constructor_with_explicit_project(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes, project=PROJECT) - - def test_constructor_with_explicit_user_agent(self): - from gcloud_bigtable import client as MUT - user_agent = 'USER_AGENT' - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes, user_agent=user_agent) - - def test_constructor_with_admin(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE, MUT.ADMIN_SCOPE] - self._constructor_test_helper(expected_scopes, admin=True) - - def test_constructor_with_read_only(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.READ_ONLY_SCOPE] - self._constructor_test_helper(expected_scopes, read_only=True) - - def test_constructor_both_admin_and_read_only(self): - with self.assertRaises(ValueError): - self._makeOne(project=PROJECT, credentials=None, - admin=True, read_only=True) - - def _copy_test_helper(self, read_only=False, admin=False): - class Credentials(object): - - scopes = None - - def __init__(self, value=None): - self.value = value - - def create_scoped(self, scope): - self.scopes = scope - return self - - def __eq__(self, other): - return self.value == other.value - - credentials = Credentials('value') - timeout_seconds = 123 - user_agent = 'you-sir-age-int' - client = self._makeOne(project=PROJECT, credentials=credentials, - read_only=read_only, admin=admin, - timeout_seconds=timeout_seconds, - user_agent=user_agent) - # Put some fake stubs in place so that we can verify they - # don't get copied. - client._data_stub_internal = object() - client._cluster_stub_internal = object() - client._operations_stub_internal = object() - client._table_stub_internal = object() - - new_client = client.copy() - self.assertEqual(new_client._admin, client._admin) - self.assertEqual(new_client._credentials, client._credentials) - # Make sure credentials (a non-simple type) gets copied - # to a new instance. - self.assertFalse(new_client._credentials is client._credentials) - self.assertEqual(new_client.project, client.project) - self.assertEqual(new_client.user_agent, client.user_agent) - self.assertEqual(new_client.timeout_seconds, client.timeout_seconds) - # Make sure stubs are not preserved. - self.assertEqual(new_client._data_stub_internal, None) - self.assertEqual(new_client._cluster_stub_internal, None) - self.assertEqual(new_client._operations_stub_internal, None) - self.assertEqual(new_client._table_stub_internal, None) - - def test_copy(self): - self._copy_test_helper() - - def test_copy_admin(self): - self._copy_test_helper(admin=True) - - def test_copy_read_only(self): - self._copy_test_helper(read_only=True) - - def test_credentials_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertTrue(client.credentials is scoped_creds) - - def test_project_name_property(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - project_name = 'projects/' + PROJECT - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertEqual(client.project_name, project_name) - - def test_data_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - client._data_stub_internal = object() - self.assertTrue(client._data_stub is client._data_stub_internal) - - def test_data_stub_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - with self.assertRaises(ValueError): - getattr(client, '_data_stub') - - def test_cluster_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._cluster_stub_internal = object() - self.assertTrue(client._cluster_stub is client._cluster_stub_internal) - - def test_cluster_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_cluster_stub') - - def test_cluster_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_cluster_stub') - - def test_operations_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._operations_stub_internal = object() - self.assertTrue(client._operations_stub is - client._operations_stub_internal) - - def test_operations_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_operations_stub') - - def test_operations_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_operations_stub') - - def test_table_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._table_stub_internal = object() - self.assertTrue(client._table_stub is client._table_stub_internal) - - def test_table_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_table_stub') - - def test_table_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_table_stub') - - def test__make_data_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import DATA_API_HOST - from gcloud_bigtable.client import DATA_API_PORT - from gcloud_bigtable.client import DATA_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, _make_stub=mock_make_stub): - result = client._make_data_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - DATA_STUB_FACTORY, - DATA_API_HOST, - DATA_API_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_cluster_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import CLUSTER_ADMIN_HOST - from gcloud_bigtable.client import CLUSTER_ADMIN_PORT - from gcloud_bigtable.client import CLUSTER_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, _make_stub=mock_make_stub): - result = client._make_cluster_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, - CLUSTER_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_operations_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import CLUSTER_ADMIN_HOST - from gcloud_bigtable.client import CLUSTER_ADMIN_PORT - from gcloud_bigtable.client import OPERATIONS_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, _make_stub=mock_make_stub): - result = client._make_operations_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, - CLUSTER_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_table_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import TABLE_ADMIN_HOST - from gcloud_bigtable.client import TABLE_ADMIN_PORT - from gcloud_bigtable.client import TABLE_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, _make_stub=mock_make_stub): - result = client._make_table_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, - TABLE_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test_is_started(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertFalse(client.is_started()) - client._data_stub_internal = object() - self.assertTrue(client.is_started()) - client._data_stub_internal = None - self.assertFalse(client.is_started()) - - def _start_method_helper(self, admin): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=admin) - stub = _FakeStub() - mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, _make_stub=mock_make_stub): - client.start() - - self.assertTrue(client._data_stub_internal is stub) - if admin: - self.assertTrue(client._cluster_stub_internal is stub) - self.assertTrue(client._operations_stub_internal is stub) - self.assertTrue(client._table_stub_internal is stub) - self.assertEqual(stub._entered, 4) - else: - self.assertTrue(client._cluster_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub._entered, 1) - self.assertEqual(stub._exited, []) - - def test_start_non_admin(self): - self._start_method_helper(admin=False) - - def test_start_with_admin(self): - self._start_method_helper(admin=True) - - def test_start_while_started(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - client._data_stub_internal = data_stub = object() - self.assertTrue(client.is_started()) - client.start() - - # Make sure the stub did not change. - self.assertEqual(client._data_stub_internal, data_stub) - - def _stop_method_helper(self, admin): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=admin) - stub1 = _FakeStub() - stub2 = _FakeStub() - client._data_stub_internal = stub1 - client._cluster_stub_internal = stub2 - client._operations_stub_internal = stub2 - client._table_stub_internal = stub2 - client.stop() - self.assertTrue(client._data_stub_internal is None) - self.assertTrue(client._cluster_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub1._entered, 0) - self.assertEqual(stub2._entered, 0) - exc_none_triple = (None, None, None) - self.assertEqual(stub1._exited, [exc_none_triple]) - if admin: - self.assertEqual(stub2._exited, [exc_none_triple] * 3) - else: - self.assertEqual(stub2._exited, []) - - def test_stop_non_admin(self): - self._stop_method_helper(admin=False) - - def test_stop_with_admin(self): - self._stop_method_helper(admin=True) - - def test_stop_while_stopped(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertFalse(client.is_started()) - - # This is a bit hacky. We set the cluster stub protected value - # since it isn't used in is_started() and make sure that stop - # doesn't reset this value to None. - client._cluster_stub_internal = cluster_stub = object() - client.stop() - # Make sure the cluster stub did not change. - self.assertEqual(client._cluster_stub_internal, cluster_stub) - - def test_cluster_factory(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.cluster import Cluster - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - - zone = 'zone' - cluster_id = 'cluster-id' - cluster = client.cluster(zone, cluster_id) - self.assertTrue(isinstance(cluster, Cluster)) - self.assertTrue(cluster._client is client) - self.assertEqual(cluster.zone, zone) - self.assertEqual(cluster.cluster_id, cluster_id) - - def _list_zones_helper(self, zone_status): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - timeout_seconds = 281330 - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True, - timeout_seconds=timeout_seconds) - - # Create request_pb - request_pb = messages_pb2.ListZonesRequest( - name='projects/' + PROJECT, - ) - - # Create response_pb - zone1 = 'foo' - zone2 = 'bar' - response_pb = messages_pb2.ListZonesResponse( - zones=[ - data_pb2.Zone(display_name=zone1, status=zone_status), - data_pb2.Zone(display_name=zone2, status=zone_status), - ], - ) - - # Patch the stub used by the API method. - client._cluster_stub_internal = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = [zone1, zone2] - - # Perform the method and check the result. - result = client.list_zones() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListZones', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_zones(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - self._list_zones_helper(data_pb2.Zone.OK) - - def test_list_zones_failure(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - with self.assertRaises(ValueError): - self._list_zones_helper(data_pb2.Zone.EMERGENCY_MAINENANCE) - - def test_list_clusters(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - timeout_seconds = 8004 - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True, - timeout_seconds=timeout_seconds) - - # Create request_pb - request_pb = messages_pb2.ListClustersRequest( - name='projects/' + PROJECT, - ) - - # Create response_pb - zone = 'foo' - failed_zone = 'bar' - cluster_id1 = 'cluster-id1' - cluster_id2 = 'cluster-id2' - cluster_name1 = ('projects/' + PROJECT + '/zones/' + zone + - '/clusters/' + cluster_id1) - cluster_name2 = ('projects/' + PROJECT + '/zones/' + zone + - '/clusters/' + cluster_id2) - response_pb = messages_pb2.ListClustersResponse( - failed_zones=[ - data_pb2.Zone(display_name=failed_zone), - ], - clusters=[ - data_pb2.Cluster( - name=cluster_name1, - display_name=cluster_name1, - serve_nodes=3, - ), - data_pb2.Cluster( - name=cluster_name2, - display_name=cluster_name2, - serve_nodes=3, - ), - ], - ) - - # Patch the stub used by the API method. - client._cluster_stub_internal = stub = StubMock(response_pb) - - # Create expected_result. - failed_zones = [failed_zone] - clusters = [ - client.cluster(zone, cluster_id1), - client.cluster(zone, cluster_id2), - ] - expected_result = (clusters, failed_zones) - - # Perform the method and check the result. - result = client.list_clusters() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListClusters', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test_MetadataPlugin(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.client import _MetadataPlugin - return _MetadataPlugin - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - user_agent = 'USER_AGENT' - client = Client(project=project, credentials=credentials, - user_agent=user_agent) - plugin = self._makeOne(client) - self.assertTrue(plugin._credentials is scoped_creds) - self.assertEqual(plugin._user_agent, user_agent) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - - def test___call__(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - from gcloud_bigtable.client import DEFAULT_USER_AGENT - - access_token_expected = 'FOOBARBAZ' - - class _ReturnVal(object): - access_token = access_token_expected - - callback_args = [] - - def callback(value1, value2): - callback_args.append((value1, value2)) - - scoped_creds = _MockWithAttachedMethods(_ReturnVal) - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - client = Client(project=project, credentials=credentials) - - plugin = self._makeOne(client) - result = plugin(None, callback) - self.assertEqual(result, None) - cb_headers = [ - ('Authorization', 'Bearer ' + access_token_expected), - ('User-agent', DEFAULT_USER_AGENT), - ] - self.assertEqual(callback_args, [(cb_headers, None)]) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) - - -class Test__make_stub(unittest2.TestCase): - - def _callFUT(self, credentials, stub_factory, host, port): - from gcloud_bigtable.client import _make_stub - return _make_stub(credentials, stub_factory, host, port) - - def test_it(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - mock_result = object() - custom_factory = _MockCalled(mock_result) - mock_plugin = object() - plugin = _MockCalled(mock_plugin) - - ssl_creds = object() - metadata_creds = object() - composite_creds = object() - channel = object() - implementations_mod = _MockWithAttachedMethods( - ssl_creds, metadata_creds, composite_creds, channel) - - host = 'HOST' - port = 1025 - client = _MockWithAttachedMethods() - with _Monkey(MUT, implementations=implementations_mod, - _MetadataPlugin=plugin): - result = self._callFUT(client, custom_factory, host, port) - - self.assertTrue(result is mock_result) - custom_factory.check_called( - self, - [(channel,)], - [{}], - ) - plugin.check_called(self, [(client,)]) - self.assertEqual(client._called, []) - # Check what was called on the module. - self.assertEqual(implementations_mod._called, [ - ('ssl_channel_credentials', (None, None, None), {}), - ('metadata_call_credentials', (mock_plugin,), - {'name': 'google_creds'}), - ('composite_channel_credentials', - (ssl_creds, metadata_creds), {}), - ('secure_channel', (host, port, composite_creds), {}), - ]) - - -class _FakeStub(object): - - def __init__(self): - self._entered = 0 - self._exited = [] - - def __enter__(self): - self._entered += 1 - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._exited.append((exc_type, exc_val, exc_tb)) - return True diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py deleted file mode 100644 index aef4fa5..0000000 --- a/gcloud_bigtable/test_cluster.py +++ /dev/null @@ -1,761 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' - - -class Test__prepare_create_request(unittest2.TestCase): - - def _callFUT(self, cluster): - from gcloud_bigtable.cluster import _prepare_create_request - return _prepare_create_request(cluster) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.cluster import Cluster - - display_name = u'DISPLAY_NAME' - serve_nodes = 8 - - cluster = Cluster(ZONE, CLUSTER_ID, _Client(PROJECT), - display_name=display_name, serve_nodes=serve_nodes) - request_pb = self._callFUT(cluster) - self.assertTrue(isinstance(request_pb, - messages_pb2.CreateClusterRequest)) - self.assertEqual(request_pb.cluster_id, CLUSTER_ID) - self.assertEqual(request_pb.name, - 'projects/' + PROJECT + '/zones/' + ZONE) - self.assertTrue(isinstance(request_pb.cluster, data_pb2.Cluster)) - self.assertEqual(request_pb.cluster.display_name, display_name) - self.assertEqual(request_pb.cluster.serve_nodes, serve_nodes) - - -class Test__process_operation(unittest2.TestCase): - - def _callFUT(self, operation_pb): - from gcloud_bigtable.cluster import _process_operation - return _process_operation(operation_pb) - - def test_it(self): - from google.longrunning import operations_pb2 - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - expected_operation_id = 234 - operation_name = ('operations/projects/%s/zones/%s/clusters/%s/' - 'operations/%d' % (PROJECT, ZONE, CLUSTER_ID, - expected_operation_id)) - - current_op = operations_pb2.Operation(name=operation_name) - - request_metadata = messages_pb2.CreateClusterMetadata() - mock_parse_pb_any_to_native = _MockCalled(request_metadata) - expected_operation_begin = object() - mock_pb_timestamp_to_datetime = _MockCalled(expected_operation_begin) - with _Monkey(MUT, _parse_pb_any_to_native=mock_parse_pb_any_to_native, - _pb_timestamp_to_datetime=mock_pb_timestamp_to_datetime): - operation_id, operation_begin = self._callFUT(current_op) - - self.assertEqual(operation_id, expected_operation_id) - self.assertTrue(operation_begin is expected_operation_begin) - - mock_parse_pb_any_to_native.check_called( - self, [(current_op.metadata,)]) - mock_pb_timestamp_to_datetime.check_called( - self, [(request_metadata.request_time,)]) - - def test_failure(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - - cluster = data_pb2.Cluster() - with self.assertRaises(ValueError): - self._callFUT(cluster) - - -class TestOperation(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.cluster import Operation - return Operation - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _constructor_test_helper(self, cluster=None): - import datetime - op_type = 'fake-op' - op_id = 8915 - begin = datetime.datetime(2015, 10, 22, 1, 1) - operation = self._makeOne(op_type, op_id, begin, cluster=cluster) - - self.assertEqual(operation.op_type, op_type) - self.assertEqual(operation.op_id, op_id) - self.assertEqual(operation.begin, begin) - self.assertEqual(operation._cluster, cluster) - self.assertFalse(operation._complete) - - def test_constructor_defaults(self): - self._constructor_test_helper() - - def test_constructor_explicit_cluster(self): - cluster = object() - self._constructor_test_helper(cluster=cluster) - - def test___eq__(self): - import datetime - op_type = 'fake-op' - op_id = 8915 - begin = datetime.datetime(2015, 10, 22, 1, 1) - cluster = object() - operation1 = self._makeOne(op_type, op_id, begin, cluster=cluster) - operation2 = self._makeOne(op_type, op_id, begin, cluster=cluster) - self.assertEqual(operation1, operation2) - - def test___eq__type_differ(self): - operation1 = self._makeOne('foo', 123, None) - operation2 = object() - self.assertNotEqual(operation1, operation2) - - def test___ne__same_value(self): - import datetime - op_type = 'fake-op' - op_id = 8915 - begin = datetime.datetime(2015, 10, 22, 1, 1) - cluster = object() - operation1 = self._makeOne(op_type, op_id, begin, cluster=cluster) - operation2 = self._makeOne(op_type, op_id, begin, cluster=cluster) - comparison_val = (operation1 != operation2) - self.assertFalse(comparison_val) - - def test___ne__(self): - operation1 = self._makeOne('foo', 123, None) - operation2 = self._makeOne('bar', 456, None) - self.assertNotEqual(operation1, operation2) - - def test_finished_without_operation(self): - operation = self._makeOne(None, None, None) - operation._complete = True - with self.assertRaises(ValueError): - operation.finished() - - def _finished_helper(self, done): - import datetime - from google.longrunning import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.cluster import Cluster - - project = 'PROJECT' - zone = 'zone' - cluster_id = 'cluster-id' - op_type = 'fake-op' - op_id = 789 - begin = datetime.datetime(2015, 10, 22, 1, 1) - timeout_seconds = 1 - - client = _Client(project, timeout_seconds=timeout_seconds) - cluster = Cluster(zone, cluster_id, client) - operation = self._makeOne(op_type, op_id, begin, cluster=cluster) - - # Create request_pb - op_name = ('operations/projects/' + project + '/zones/' + - zone + '/clusters/' + cluster_id + - '/operations/%d' % (op_id,)) - request_pb = operations_pb2.GetOperationRequest(name=op_name) - - # Create response_pb - response_pb = operations_pb2.Operation(done=done) - - # Patch the stub used by the API method. - client._operations_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = done - - # Perform the method and check the result. - result = operation.finished() - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetOperation', - (request_pb, timeout_seconds), - {}, - )]) - - if done: - self.assertTrue(operation._complete) - else: - self.assertFalse(operation._complete) - - def test_finished(self): - self._finished_helper(done=True) - - def test_finished_not_done(self): - self._finished_helper(done=False) - - -class TestCluster(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.cluster import Cluster - return Cluster - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - client = object() - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - self.assertEqual(cluster.display_name, CLUSTER_ID) - self.assertEqual(cluster.serve_nodes, 3) - self.assertTrue(cluster._client is client) - - def test_constructor_non_default(self): - client = object() - display_name = 'display_name' - serve_nodes = 8 - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - self.assertEqual(cluster.display_name, display_name) - self.assertEqual(cluster.serve_nodes, serve_nodes) - self.assertTrue(cluster._client is client) - - def test_copy(self): - client = _Client(PROJECT) - display_name = 'DISPLAY_NAME' - serve_nodes = 8 - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - new_cluster = cluster.copy() - - # Make sure the client got copied to a new instance. - self.assertFalse(new_cluster._client is client) - self.assertEqual(new_cluster._client.__dict__, - client.__dict__) - # Just replace the client on the new_cluster so we can - # check cluster equality. - new_cluster._client = client - self.assertFalse(cluster is new_cluster) - self.assertEqual(cluster, new_cluster) - - def test_name_property(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - self.assertEqual(cluster.name, cluster_name) - - def test_table_factory(self): - from gcloud_bigtable.table import Table - - cluster = self._makeOne(ZONE, CLUSTER_ID, None) - table_id = 'table_id' - table = cluster.table(table_id) - self.assertTrue(isinstance(table, Table)) - self.assertEqual(table.table_id, table_id) - self.assertEqual(table._cluster, cluster) - - def test__update_from_pb_no_display_name(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable.cluster import _DEFAULT_SERVE_NODES - - cluster_pb = data_pb2.Cluster(serve_nodes=331) - cluster = self._makeOne(None, None, None) - self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) - with self.assertRaises(ValueError): - cluster._update_from_pb(cluster_pb) - self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) - - def test__update_from_pb_no_serve_nodes(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable.cluster import _DEFAULT_SERVE_NODES - - cluster_pb = data_pb2.Cluster(display_name='name') - cluster = self._makeOne(None, None, None) - self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) - with self.assertRaises(ValueError): - cluster._update_from_pb(cluster_pb) - self.assertEqual(cluster.display_name, None) - self.assertEqual(cluster.serve_nodes, _DEFAULT_SERVE_NODES) - - def test_from_pb_success(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster_pb = data_pb2.Cluster( - name=cluster_name, - display_name=CLUSTER_ID, - serve_nodes=3, - ) - - klass = self._getTargetClass() - cluster = klass.from_pb(cluster_pb, client) - self.assertTrue(isinstance(cluster, klass)) - self.assertEqual(cluster._client, client) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - - def test_from_pb_bad_cluster_name(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - - cluster_name = 'INCORRECT_FORMAT' - cluster_pb = data_pb2.Cluster(name=cluster_name) - - klass = self._getTargetClass() - with self.assertRaises(ValueError): - klass.from_pb(cluster_pb, None) - - def test_from_pb_project_mistmatch(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - alt_project = 'ALT_PROJECT' - client = Client(project=alt_project, credentials=credentials) - - self.assertNotEqual(PROJECT, alt_project) - - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster_pb = data_pb2.Cluster(name=cluster_name) - - klass = self._getTargetClass() - with self.assertRaises(ValueError): - klass.from_pb(cluster_pb, client) - - def test___eq__(self): - zone = 'zone' - cluster_id = 'cluster_id' - client = object() - cluster1 = self._makeOne(zone, cluster_id, client) - cluster2 = self._makeOne(zone, cluster_id, client) - self.assertEqual(cluster1, cluster2) - - def test___eq__type_differ(self): - cluster1 = self._makeOne('zone', 'cluster_id', 'client') - cluster2 = object() - self.assertNotEqual(cluster1, cluster2) - - def test___ne__same_value(self): - zone = 'zone' - cluster_id = 'cluster_id' - client = object() - cluster1 = self._makeOne(zone, cluster_id, client) - cluster2 = self._makeOne(zone, cluster_id, client) - comparison_val = (cluster1 != cluster2) - self.assertFalse(comparison_val) - - def test___ne__(self): - cluster1 = self._makeOne('zone1', 'cluster_id1', 'client1') - cluster2 = self._makeOne('zone2', 'cluster_id2', 'client2') - self.assertNotEqual(cluster1, cluster2) - - def test_reload(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 123 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.GetClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = data_pb2.Cluster( - display_name=CLUSTER_ID, - serve_nodes=3, - ) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # reload() has no return value. - - # Perform the method and check the result. - result = cluster.reload() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetCluster', - (request_pb, timeout_seconds), - {}, - )]) - - def test_create(self): - from google.longrunning import operations_pb2 - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - timeout_seconds = 578 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb. Just a mock since we monkey patch - # _prepare_create_request - request_pb = object() - - # Create response_pb - current_op = operations_pb2.Operation() - response_pb = data_pb2.Cluster(current_operation=current_op) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - op_id = 5678 - op_begin = object() - expected_result = MUT.Operation('create', op_id, op_begin, - cluster=cluster) - - # Perform the method and check the result. - mock_prepare_create_request = _MockCalled(request_pb) - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, _prepare_create_request=mock_prepare_create_request, - _process_operation=mock_process_operation): - result = cluster.create() - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateCluster', - (request_pb, timeout_seconds), - {}, - )]) - mock_prepare_create_request.check_called(self, [(cluster,)]) - mock_process_operation.check_called(self, [(current_op,)]) - - def test_update(self): - from google.longrunning import operations_pb2 - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - timeout_seconds = 9 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - serve_nodes = 81 - display_name = 'display_name' - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = data_pb2.Cluster( - name=cluster_name, - display_name=display_name, - serve_nodes=serve_nodes, - ) - - # Create response_pb - current_op = operations_pb2.Operation() - response_pb = data_pb2.Cluster(current_operation=current_op) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - op_id = 5678 - op_begin = object() - expected_result = MUT.Operation('update', op_id, op_begin, - cluster=cluster) - - # We must create the cluster object with the client passed in. - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, - _process_operation=mock_process_operation): - result = cluster.update() - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UpdateCluster', - (request_pb, timeout_seconds), - {}, - )]) - mock_process_operation.check_called(self, [(current_op,)]) - - def test_delete(self): - from google.protobuf import empty_pb2 - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 57 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.DeleteClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - result = cluster.delete() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteCluster', - (request_pb, timeout_seconds), - {}, - )]) - - def test_undelete(self): - from google.longrunning import operations_pb2 - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - timeout_seconds = 78 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.UndeleteClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = operations_pb2.Operation() - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - op_id = 5678 - op_begin = object() - expected_result = MUT.Operation('undelete', op_id, op_begin, - cluster=cluster) - - # Perform the method and check the result. - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, - _process_operation=mock_process_operation): - result = cluster.undelete() - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UndeleteCluster', - (request_pb, timeout_seconds), - {}, - )]) - mock_process_operation.check_called(self, [(response_pb,)]) - - def _list_tables_helper(self, table_id, table_name=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as table_data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as table_messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 45 - client = _Client(PROJECT, timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_ - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = table_messages_pb2.ListTablesRequest(name=cluster_name) - - # Create response_pb - table_name = table_name or (cluster_name + '/tables/' + table_id) - response_pb = table_messages_pb2.ListTablesResponse( - tables=[ - table_data_pb2.Table(name=table_name), - ], - ) - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_table = cluster.table(table_id) - expected_result = [expected_table] - - # Perform the method and check the result. - result = cluster.list_tables() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListTables', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_tables(self): - table_id = 'table_id' - self._list_tables_helper(table_id) - - def test_list_tables_failure_bad_split(self): - with self.assertRaises(ValueError): - self._list_tables_helper(None, table_name='wrong-format') - - def test_list_tables_failure_name_bad_before(self): - table_id = 'table_id' - bad_table_name = ('nonempty-section-before' + - 'projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + table_id) - with self.assertRaises(ValueError): - self._list_tables_helper(table_id, table_name=bad_table_name) - - -class Test__pb_timestamp_to_datetime(unittest2.TestCase): - - def _callFUT(self, timestamp): - from gcloud_bigtable.cluster import _pb_timestamp_to_datetime - return _pb_timestamp_to_datetime(timestamp) - - def test_it(self): - import datetime - import pytz - from google.protobuf.timestamp_pb2 import Timestamp - - # Epoch is midnight on January 1, 1970 ... - dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0, - minute=1, second=1, microsecond=1234, - tzinfo=pytz.utc) - # ... so 1 minute and 1 second after is 61 seconds and 1234 - # microseconds is 1234000 nanoseconds. - timestamp = Timestamp(seconds=61, nanos=1234000) - self.assertEqual(self._callFUT(timestamp), dt_stamp) - - -class Test__parse_pb_any_to_native(unittest2.TestCase): - - def _callFUT(self, any_val, expected_type=None): - from gcloud_bigtable.cluster import _parse_pb_any_to_native - return _parse_pb_any_to_native(any_val, expected_type=expected_type) - - def test_it(self): - from google.protobuf import any_pb2 - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name - fake_type_url_map = {type_url: data_pb2.Cell} - - cell = data_pb2.Cell( - timestamp_micros=0, - value=b'foobar', - ) - any_val = any_pb2.Any( - type_url=type_url, - value=cell.SerializeToString(), - ) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - result = self._callFUT(any_val) - - self.assertEqual(result, cell) - - def test_unknown_type_url(self): - from google.protobuf import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - fake_type_url_map = {} - any_val = any_pb2.Any() - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(KeyError): - self._callFUT(any_val) - - def test_disagreeing_type_url(self): - from google.protobuf import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - type_url1 = 'foo' - type_url2 = 'bar' - fake_type_url_map = {type_url1: None} - any_val = any_pb2.Any(type_url=type_url2) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(ValueError): - self._callFUT(any_val, expected_type=type_url1) - - -class _Client(object): - - cluster_stub = None - operations_stub = None - table_stub = None - - def __init__(self, project, timeout_seconds=None): - self.project = project - self.project_name = 'projects/' + project - self.timeout_seconds = timeout_seconds - - def copy(self): - import copy as copy_module - return copy_module.deepcopy(self) diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py deleted file mode 100644 index 05fe569..0000000 --- a/gcloud_bigtable/test_column_family.py +++ /dev/null @@ -1,627 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' -TABLE_ID = 'table-id' -COLUMN_FAMILY_ID = 'column-family-id' - - -class TestMaxVersionsGCRule(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - return MaxVersionsGCRule - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test___eq__type_differ(self): - gc_rule1 = self._makeOne(10) - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___eq__same_value(self): - gc_rule1 = self._makeOne(2) - gc_rule2 = self._makeOne(2) - self.assertEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - gc_rule1 = self._makeOne(99) - gc_rule2 = self._makeOne(99) - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - max_num_versions = 1337 - gc_rule = self._makeOne(max_num_versions=max_num_versions) - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, - data_pb2.GcRule(max_num_versions=max_num_versions)) - - -class TestMaxAgeGCRule(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import MaxAgeGCRule - return MaxAgeGCRule - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test___eq__type_differ(self): - max_age = object() - gc_rule1 = self._makeOne(max_age=max_age) - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___eq__same_value(self): - max_age = object() - gc_rule1 = self._makeOne(max_age=max_age) - gc_rule2 = self._makeOne(max_age=max_age) - self.assertEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - max_age = object() - gc_rule1 = self._makeOne(max_age=max_age) - gc_rule2 = self._makeOne(max_age=max_age) - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - import datetime - from google.protobuf import duration_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - - max_age = datetime.timedelta(seconds=1) - duration = duration_pb2.Duration(seconds=1) - gc_rule = self._makeOne(max_age=max_age) - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, data_pb2.GcRule(max_age=duration)) - - -class TestGCRuleUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import GCRuleUnion - return GCRuleUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - rules = object() - rule_union = self._makeOne(rules) - self.assertTrue(rule_union.rules is rules) - - def test___eq__(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = self._makeOne(rules) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__type_differ(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = self._makeOne(rules) - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - import datetime - from google.protobuf import duration_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - max_num_versions = 42 - rule1 = MaxVersionsGCRule(max_num_versions) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) - - max_age = datetime.timedelta(seconds=1) - rule2 = MaxAgeGCRule(max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule1, pb_rule2])) - - gc_rule_pb = rule3.to_pb() - self.assertEqual(gc_rule_pb, pb_rule3) - - def test_to_pb_nested(self): - import datetime - from google.protobuf import duration_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - max_num_versions1 = 42 - rule1 = MaxVersionsGCRule(max_num_versions1) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) - - max_age = datetime.timedelta(seconds=1) - rule2 = MaxAgeGCRule(max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule1, pb_rule2])) - - max_num_versions2 = 1337 - rule4 = MaxVersionsGCRule(max_num_versions2) - pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) - - rule5 = self._makeOne(rules=[rule3, rule4]) - pb_rule5 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule3, pb_rule4])) - - gc_rule_pb = rule5.to_pb() - self.assertEqual(gc_rule_pb, pb_rule5) - - -class TestGCRuleIntersection(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import GCRuleIntersection - return GCRuleIntersection - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - rules = object() - rule_intersection = self._makeOne(rules) - self.assertTrue(rule_intersection.rules is rules) - - def test___eq__(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = self._makeOne(rules) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__type_differ(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - rules = object() - gc_rule1 = self._makeOne(rules) - gc_rule2 = self._makeOne(rules) - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - import datetime - from google.protobuf import duration_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - max_num_versions = 42 - rule1 = MaxVersionsGCRule(max_num_versions) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) - - max_age = datetime.timedelta(seconds=1) - rule2 = MaxAgeGCRule(max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule1, pb_rule2])) - - gc_rule_pb = rule3.to_pb() - self.assertEqual(gc_rule_pb, pb_rule3) - - def test_to_pb_nested(self): - import datetime - from google.protobuf import duration_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - max_num_versions1 = 42 - rule1 = MaxVersionsGCRule(max_num_versions1) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) - - max_age = datetime.timedelta(seconds=1) - rule2 = MaxAgeGCRule(max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule1, pb_rule2])) - - max_num_versions2 = 1337 - rule4 = MaxVersionsGCRule(max_num_versions2) - pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) - - rule5 = self._makeOne(rules=[rule3, rule4]) - pb_rule5 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule3, pb_rule4])) - - gc_rule_pb = rule5.to_pb() - self.assertEqual(gc_rule_pb, pb_rule5) - - -class TestColumnFamily(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import ColumnFamily - return ColumnFamily - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - table = object() - gc_rule = object() - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - self.assertEqual(column_family.column_family_id, COLUMN_FAMILY_ID) - self.assertTrue(column_family._table is table) - self.assertTrue(column_family.gc_rule is gc_rule) - - def test_name_property(self): - table_name = 'table_name' - table = _Table(table_name) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - expected_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - self.assertEqual(column_family.name, expected_name) - - def test___eq__(self): - column_family_id = 'column_family_id' - table = object() - column_family1 = self._makeOne(column_family_id, table) - column_family2 = self._makeOne(column_family_id, table) - self.assertEqual(column_family1, column_family2) - - def test___eq__type_differ(self): - column_family1 = self._makeOne('column_family_id', None) - column_family2 = object() - self.assertNotEqual(column_family1, column_family2) - - def test___ne__same_value(self): - column_family_id = 'column_family_id' - table = object() - column_family1 = self._makeOne(column_family_id, table) - column_family2 = self._makeOne(column_family_id, table) - comparison_val = (column_family1 != column_family2) - self.assertFalse(comparison_val) - - def test___ne__(self): - column_family1 = self._makeOne('column_family_id1', 'table1') - column_family2 = self._makeOne('column_family_id2', 'table2') - self.assertNotEqual(column_family1, column_family2) - - def _create_test_helper(self, gc_rule=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 4 - client = _Client(timeout_seconds=timeout_seconds) - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - - # Create request_pb - if gc_rule is None: - column_family_pb = data_pb2.ColumnFamily() - else: - column_family_pb = data_pb2.ColumnFamily(gc_rule=gc_rule.to_pb()) - request_pb = messages_pb2.CreateColumnFamilyRequest( - name=table_name, - column_family_id=COLUMN_FAMILY_ID, - column_family=column_family_pb, - ) - - # Create response_pb - response_pb = data_pb2.ColumnFamily() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # create() has no return value. - - # Perform the method and check the result. - result = column_family.create() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - def test_create(self): - self._create_test_helper(gc_rule=None) - - def test_create_with_gc_rule(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - gc_rule = MaxVersionsGCRule(1337) - self._create_test_helper(gc_rule=gc_rule) - - def _update_test_helper(self, gc_rule=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 28 - client = _Client(timeout_seconds=timeout_seconds) - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - - # Create request_pb - column_family_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - if gc_rule is None: - request_pb = data_pb2.ColumnFamily(name=column_family_name) - else: - request_pb = data_pb2.ColumnFamily( - name=column_family_name, - gc_rule=gc_rule.to_pb(), - ) - - # Create response_pb - response_pb = data_pb2.ColumnFamily() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # update() has no return value. - - # Perform the method and check the result. - result = column_family.update() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UpdateColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - def test_update(self): - self._update_test_helper(gc_rule=None) - - def test_update_with_gc_rule(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - gc_rule = MaxVersionsGCRule(1337) - self._update_test_helper(gc_rule=gc_rule) - - def test_delete(self): - from google.protobuf import empty_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 7 - client = _Client(timeout_seconds=timeout_seconds) - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - - # Create request_pb - column_family_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - request_pb = messages_pb2.DeleteColumnFamilyRequest( - name=column_family_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - result = column_family.delete() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test__gc_rule_from_pb(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.column_family import _gc_rule_from_pb - return _gc_rule_from_pb(*args, **kwargs) - - def test_empty(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - - gc_rule_pb = data_pb2.GcRule() - self.assertEqual(self._callFUT(gc_rule_pb), None) - - def test_max_num_versions(self): - from gcloud_bigtable.column_family import MaxVersionsGCRule - - orig_rule = MaxVersionsGCRule(1) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, MaxVersionsGCRule)) - self.assertEqual(result, orig_rule) - - def test_max_age(self): - import datetime - from gcloud_bigtable.column_family import MaxAgeGCRule - - orig_rule = MaxAgeGCRule(datetime.timedelta(seconds=1)) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, MaxAgeGCRule)) - self.assertEqual(result, orig_rule) - - def test_union(self): - import datetime - from gcloud_bigtable.column_family import GCRuleUnion - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - rule1 = MaxVersionsGCRule(1) - rule2 = MaxAgeGCRule(datetime.timedelta(seconds=1)) - orig_rule = GCRuleUnion([rule1, rule2]) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GCRuleUnion)) - self.assertEqual(result, orig_rule) - - def test_intersection(self): - import datetime - from gcloud_bigtable.column_family import GCRuleIntersection - from gcloud_bigtable.column_family import MaxAgeGCRule - from gcloud_bigtable.column_family import MaxVersionsGCRule - - rule1 = MaxVersionsGCRule(1) - rule2 = MaxAgeGCRule(datetime.timedelta(seconds=1)) - orig_rule = GCRuleIntersection([rule1, rule2]) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GCRuleIntersection)) - self.assertEqual(result, orig_rule) - - def test_unknown_field_name(self): - class MockProto(object): - - names = [] - - @classmethod - def WhichOneof(cls, name): - cls.names.append(name) - return 'unknown' - - self.assertEqual(MockProto.names, []) - self.assertRaises(ValueError, self._callFUT, MockProto) - self.assertEqual(MockProto.names, ['rule']) - - -class Test__timedelta_to_duration_pb(unittest2.TestCase): - - def _callFUT(self, timedelta_val): - from gcloud_bigtable.column_family import _timedelta_to_duration_pb - return _timedelta_to_duration_pb(timedelta_val) - - def test_it(self): - import datetime - from google.protobuf import duration_pb2 - - seconds = microseconds = 1 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds) - self.assertEqual(result.nanos, 1000 * microseconds) - - def test_with_negative_microseconds(self): - import datetime - from google.protobuf import duration_pb2 - - seconds = 1 - microseconds = -5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds - 1) - self.assertEqual(result.nanos, 10**9 + 1000 * microseconds) - - def test_with_negative_seconds(self): - import datetime - from google.protobuf import duration_pb2 - - seconds = -1 - microseconds = 5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds + 1) - self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds)) - - -class Test__duration_pb_to_timedelta(unittest2.TestCase): - - def _callFUT(self, duration_pb): - from gcloud_bigtable.column_family import _duration_pb_to_timedelta - return _duration_pb_to_timedelta(duration_pb) - - def test_it(self): - import datetime - from google.protobuf import duration_pb2 - - seconds = microseconds = 1 - duration_pb = duration_pb2.Duration(seconds=seconds, - nanos=1000 * microseconds) - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(duration_pb) - self.assertTrue(isinstance(result, datetime.timedelta)) - self.assertEqual(result, timedelta_val) - - -class _Client(object): - - cluster_stub = None - operations_stub = None - table_stub = None - - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds - - -class _Cluster(object): - - def __init__(self, client=None): - self._client = client - - -class _Table(object): - - def __init__(self, name, client=None, timeout_seconds=None): - self.name = name - self._cluster = _Cluster(client) - self.timeout_seconds = timeout_seconds diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py deleted file mode 100644 index 6968744..0000000 --- a/gcloud_bigtable/test_row.py +++ /dev/null @@ -1,1735 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = u'project-id' -ZONE = u'zone' -CLUSTER_ID = u'cluster-id' -TABLE_ID = u'table-id' -TABLE_NAME = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) -ROW_KEY = b'row_key' -ROW_KEY_NON_BYTES = u'row_key' -COLUMN = b'column' -COLUMN_NON_BYTES = u'column' -COLUMN_FAMILY_ID = u'column_family_id' - - -class TestRow(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import Row - return Row - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _constructor_helper(self, row_key, row_key_expected=None, - filter_=None): - table = object() - row = self._makeOne(row_key, table, filter_=filter_) - row_key_val = row_key_expected or row_key - # Only necessary in Py2 - self.assertEqual(type(row._row_key), type(row_key_val)) - self.assertEqual(row._row_key, row_key_val) - self.assertTrue(row._table is table) - self.assertTrue(row._filter is filter_) - self.assertEqual(row._rule_pb_list, []) - if filter_ is None: - self.assertEqual(row._pb_mutations, []) - self.assertTrue(row._true_pb_mutations is None) - self.assertTrue(row._false_pb_mutations is None) - else: - self.assertTrue(row._pb_mutations is None) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - def test_constructor(self): - self._constructor_helper(ROW_KEY) - - def test_constructor_with_filter(self): - self._constructor_helper(ROW_KEY, filter_=object()) - - def test_constructor_with_unicode(self): - self._constructor_helper(ROW_KEY_NON_BYTES, row_key_expected=ROW_KEY) - - def test_constructor_with_non_bytes(self): - row_key = object() - with self.assertRaises(TypeError): - self._constructor_helper(row_key) - - def _get_mutations_helper(self, filter_=None, state=None): - row = self._makeOne(ROW_KEY, None, filter_=filter_) - # Mock the mutations with unique objects so we can compare. - row._pb_mutations = no_bool = object() - row._true_pb_mutations = true_mutations = object() - row._false_pb_mutations = false_mutations = object() - - mutations = row._get_mutations(state) - return (no_bool, true_mutations, false_mutations), mutations - - def test__get_mutations_no_filter(self): - (no_bool, _, _), mutations = self._get_mutations_helper() - self.assertTrue(mutations is no_bool) - - def test__get_mutations_no_filter_bad_state(self): - state = object() # State should be null when no filter. - with self.assertRaises(ValueError): - self._get_mutations_helper(state=state) - - def test__get_mutations_with_filter_true_state(self): - filter_ = object() - state = True - (_, true_filter, _), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is true_filter) - - def test__get_mutations_with_filter_false_state(self): - filter_ = object() - state = False - (_, _, false_filter), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is false_filter) - - def test__get_mutations_with_filter_bad_state(self): - filter_ = object() - state = None - with self.assertRaises(ValueError): - self._get_mutations_helper(filter_=filter_, state=state) - - def test_append_cell_value(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - value = b'bytes-val' - row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, column_qualifier=COLUMN, - append_value=value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - - def test_increment_cell_value(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - int_value = 281330 - row.increment_cell_value(COLUMN_FAMILY_ID, COLUMN, int_value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, column_qualifier=COLUMN, - increment_amount=int_value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - - def _set_cell_helper(self, column=COLUMN, column_bytes=None, - value=b'foobar', timestamp=None, - timestamp_micros=-1): - import six - import struct - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._pb_mutations, []) - row.set_cell(COLUMN_FAMILY_ID, column, - value, timestamp=timestamp) - - if isinstance(value, six.integer_types): - value = struct.pack('>q', value) - expected_pb = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column_bytes or column, - timestamp_micros=timestamp_micros, - value=value, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_set_cell(self): - self._set_cell_helper(column=COLUMN) - - def test_set_cell_with_string_column(self): - self._set_cell_helper(column=COLUMN_NON_BYTES, column_bytes=COLUMN) - - def test_set_cell_with_integer_value(self): - value = 1337 - self._set_cell_helper(column=COLUMN, value=value) - - def test_set_cell_with_non_bytes_value(self): - table = object() - row = self._makeOne(ROW_KEY, table) - value = object() # Not bytes - with self.assertRaises(TypeError): - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value) - - def test_set_cell_with_non_null_timestamp(self): - import datetime - from gcloud_bigtable._non_upstream_helpers import EPOCH - - microseconds = 898294371 - millis_granularity = microseconds - (microseconds % 1000) - timestamp = EPOCH + datetime.timedelta(microseconds=microseconds) - self._set_cell_helper(timestamp=timestamp, - timestamp_micros=millis_granularity) - - def test_delete(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - row = self._makeOne(ROW_KEY, object()) - self.assertEqual(row._pb_mutations, []) - row.delete() - - expected_pb = data_pb2.Mutation( - delete_from_row=data_pb2.Mutation.DeleteFromRow(), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cell(self): - klass = self._getTargetClass() - - class MockRow(klass): - - def __init__(self, *args, **kwargs): - super(MockRow, self).__init__(*args, **kwargs) - self._args = [] - self._kwargs = [] - - # Replace the called method with one that logs arguments. - def delete_cells(self, *args, **kwargs): - self._args.append(args) - self._kwargs.append(kwargs) - - table = object() - mock_row = MockRow(ROW_KEY, table) - # Make sure no values are set before calling the method. - self.assertEqual(mock_row._pb_mutations, []) - self.assertEqual(mock_row._args, []) - self.assertEqual(mock_row._kwargs, []) - - # Actually make the request against the mock class. - time_range = object() - mock_row.delete_cell(COLUMN_FAMILY_ID, COLUMN, time_range=time_range) - self.assertEqual(mock_row._pb_mutations, []) - self.assertEqual(mock_row._args, [(COLUMN_FAMILY_ID, [COLUMN])]) - self.assertEqual(mock_row._kwargs, [{ - 'state': None, - 'time_range': time_range, - }]) - - def test_delete_cells_non_iterable(self): - table = object() - row = self._makeOne(ROW_KEY, table) - columns = object() # Not iterable - with self.assertRaises(TypeError): - row.delete_cells(COLUMN_FAMILY_ID, columns) - - def test_delete_cells_all_columns(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - klass = self._getTargetClass() - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, klass.ALL_COLUMNS) - - expected_pb = data_pb2.Mutation( - delete_from_family=data_pb2.Mutation.DeleteFromFamily( - family_name=COLUMN_FAMILY_ID, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cells_no_columns(self): - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns) - self.assertEqual(row._pb_mutations, []) - - def _delete_cells_helper(self, time_range=None): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [COLUMN] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns, time_range=time_range) - - expected_pb = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - ), - ) - if time_range is not None: - expected_pb.delete_from_column.time_range.CopyFrom( - time_range.to_pb()) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cells_no_time_range(self): - self._delete_cells_helper() - - def test_delete_cells_with_time_range(self): - import datetime - from gcloud_bigtable._non_upstream_helpers import EPOCH - from gcloud_bigtable.row import TimestampRange - - microseconds = 30871000 # Makes sure already milliseconds granularity - start = EPOCH + datetime.timedelta(microseconds=microseconds) - time_range = TimestampRange(start=start) - self._delete_cells_helper(time_range=time_range) - - def test_delete_cells_with_bad_column(self): - # This makes sure a failure on one of the columns doesn't leave - # the row's mutations in a bad state. - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [COLUMN, object()] - self.assertEqual(row._pb_mutations, []) - with self.assertRaises(TypeError): - row.delete_cells(COLUMN_FAMILY_ID, columns) - self.assertEqual(row._pb_mutations, []) - - def test_delete_cells_with_string_columns(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - column1 = u'column1' - column1_bytes = b'column1' - column2 = u'column2' - column2_bytes = b'column2' - columns = [column1, column2] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns) - - expected_pb1 = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column1_bytes, - ), - ) - expected_pb2 = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column2_bytes, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb1, expected_pb2]) - - def test_commit(self): - from google.protobuf import empty_pb2 - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 711 - client = _Client(timeout_seconds=timeout_seconds) - table = _Table(TABLE_NAME, client=client) - row = self._makeOne(ROW_KEY, table) - - # Create request_pb - value = b'bytes-value' - mutation = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - timestamp_micros=-1, # Default value. - value=value, - ), - ) - request_pb = messages_pb2.MutateRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - mutations=[mutation], - ) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # commit() has no return value when no filter. - - # Perform the method and check the result. - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'MutateRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) - - def test_commit_too_many_mutations(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - table = object() - row = self._makeOne(ROW_KEY, table) - row._pb_mutations = [1, 2, 3] - num_mutations = len(row._pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): - with self.assertRaises(ValueError): - row.commit() - - def test_commit_no_mutations(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._pb_mutations, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit() - self.assertEqual(result, None) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - def test_commit_with_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.row import RowSampleFilter - - timeout_seconds = 262 - client = _Client(timeout_seconds=timeout_seconds) - table = _Table(TABLE_NAME, client=client) - row_filter = RowSampleFilter(0.33) - row = self._makeOne(ROW_KEY, table, filter_=row_filter) - - # Create request_pb - value = b'bytes-value' - mutation = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - timestamp_micros=-1, # Default value. - value=value, - ), - ) - request_pb = messages_pb2.CheckAndMutateRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - predicate_filter=row_filter.to_pb(), - true_mutations=[mutation], - false_mutations=[], - ) - - # Create response_pb - predicate_matched = True - response_pb = messages_pb2.CheckAndMutateRowResponse( - predicate_matched=predicate_matched) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = predicate_matched - - # Perform the method and check the result. - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value, state=True) - result = row.commit() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CheckAndMutateRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, None) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - def test_commit_with_filter_too_many_mutations(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - table = object() - filter_ = object() - row = self._makeOne(ROW_KEY, table, filter_=filter_) - row._true_pb_mutations = [1, 2, 3] - num_mutations = len(row._true_pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): - with self.assertRaises(ValueError): - row.commit() - - def test_commit_with_filter_no_mutations(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - filter_ = object() - row = self._makeOne(ROW_KEY, table, filter_=filter_) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit() - self.assertEqual(result, None) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - def test_commit_modifications(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - timeout_seconds = 87 - client = _Client(timeout_seconds=timeout_seconds) - table = _Table(TABLE_NAME, client=client) - row = self._makeOne(ROW_KEY, table) - - # Create request_pb - value = b'bytes-value' - # We will call row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value). - request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - rules=[ - data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - append_value=value, - ), - ], - ) - - # Create response_pb - response_pb = object() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = object() - mock_parse_rmw_row_response = _MockCalled(expected_result) - - # Perform the method and check the result. - with _Monkey(MUT, _parse_rmw_row_response=mock_parse_rmw_row_response): - row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit_modifications() - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadModifyWriteRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) - - mock_parse_rmw_row_response.check_called(self, [(response_pb,)]) - self.assertEqual(row._rule_pb_list, []) - - def test_commit_modifications_no_rules(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit_modifications() - self.assertEqual(result, {}) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - -class Test__parse_rmw_row_response(unittest2.TestCase): - - def _callFUT(self, row_response): - from gcloud_bigtable.row import _parse_rmw_row_response - return _parse_rmw_row_response(row_response) - - def test_it(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - - COL_FAM1 = u'col-fam-id' - COL_FAM2 = u'col-fam-id2' - COL_NAME1 = b'col-name1' - COL_NAME2 = b'col-name2' - COL_NAME3 = b'col-name3-but-other-fam' - CELL_VAL1 = b'cell-val' - CELL_VAL2 = b'cell-val-newer' - CELL_VAL3 = b'altcol-cell-val' - CELL_VAL4 = b'foo' - - microseconds = 1000871 - timestamp = _microseconds_to_timestamp(microseconds) - expected_output = { - COL_FAM1: { - COL_NAME1: [ - (CELL_VAL1, timestamp), - (CELL_VAL2, timestamp), - ], - COL_NAME2: [ - (CELL_VAL3, timestamp), - ], - }, - COL_FAM2: { - COL_NAME3: [ - (CELL_VAL4, timestamp), - ], - }, - } - sample_input = data_pb2.Row( - families=[ - data_pb2.Family( - name=COL_FAM1, - columns=[ - data_pb2.Column( - qualifier=COL_NAME1, - cells=[ - data_pb2.Cell( - value=CELL_VAL1, - timestamp_micros=microseconds, - ), - data_pb2.Cell( - value=CELL_VAL2, - timestamp_micros=microseconds, - ), - ], - ), - data_pb2.Column( - qualifier=COL_NAME2, - cells=[ - data_pb2.Cell( - value=CELL_VAL3, - timestamp_micros=microseconds, - ), - ], - ), - ], - ), - data_pb2.Family( - name=COL_FAM2, - columns=[ - data_pb2.Column( - qualifier=COL_NAME3, - cells=[ - data_pb2.Cell( - value=CELL_VAL4, - timestamp_micros=microseconds, - ), - ], - ), - ], - ), - ], - ) - self.assertEqual(expected_output, self._callFUT(sample_input)) - - -class Test_BoolFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import _BoolFilter - return _BoolFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - flag = object() - row_filter = self._makeOne(flag) - self.assertTrue(row_filter.flag is flag) - - def test___eq__type_differ(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - flag = object() - row_filter1 = self._makeOne(flag) - row_filter2 = self._makeOne(flag) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestSinkFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import SinkFilter - return SinkFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(sink=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestPassAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import PassAllFilter - return PassAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(pass_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestBlockAllFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import BlockAllFilter - return BlockAllFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(block_all_filter=flag) - self.assertEqual(pb_val, expected_pb) - - -class Test_RegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import _RegexFilter - return _RegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - regex = object() - row_filter = self._makeOne(regex) - self.assertTrue(row_filter.regex is regex) - - def test___eq__type_differ(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - regex = object() - row_filter1 = self._makeOne(regex) - row_filter2 = self._makeOne(regex) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestRowKeyRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowKeyRegexFilter - return RowKeyRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'row-key-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_key_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestRowSampleFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowSampleFilter - return RowSampleFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - sample = object() - row_filter = self._makeOne(sample) - self.assertTrue(row_filter.sample is sample) - - def test___eq__type_differ(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - sample = object() - row_filter1 = self._makeOne(sample) - row_filter2 = self._makeOne(sample) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - sample = 0.25 - row_filter = self._makeOne(sample) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(row_sample_filter=sample) - self.assertEqual(pb_val, expected_pb) - - -class TestFamilyNameRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import FamilyNameRegexFilter - return FamilyNameRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = u'family-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(family_name_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnQualifierRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ColumnQualifierRegexFilter - return ColumnQualifierRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'column-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_qualifier_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestColumnRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ColumnRangeFilter - return ColumnRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - column_family_id = object() - row_filter = self._makeOne(column_family_id) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertEqual(row_filter.start_column, None) - self.assertEqual(row_filter.end_column, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.column_family_id is column_family_id) - self.assertTrue(row_filter.start_column is start_column) - self.assertTrue(row_filter.end_column is end_column) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_start=True) - - def test_constructor_bad_end(self): - column_family_id = object() - self.assertRaises(ValueError, self._makeOne, - column_family_id, inclusive_end=True) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(column_family_id, - start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - column_family_id = object() - row_filter1 = self._makeOne(column_family_id) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - row_filter = self._makeOne(column_family_id) - col_range_pb = data_pb2.ColumnRange(family_name=column_family_id) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, start_column=column, - inclusive_start=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - start_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_inclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - column_family_id = u'column-family-id' - column = b'column' - row_filter = self._makeOne(column_family_id, end_column=column, - inclusive_end=False) - col_range_pb = data_pb2.ColumnRange( - family_name=column_family_id, - end_qualifier_exclusive=column, - ) - expected_pb = data_pb2.RowFilter(column_range_filter=col_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class TestTimestampRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import TimestampRange - return TimestampRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - start = object() - end = object() - time_range = self._makeOne(start=start, end=end) - self.assertTrue(time_range.start is start) - self.assertTrue(time_range.end is end) - - def test___eq__(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - self.assertEqual(time_range1, time_range2) - - def test___eq__type_differ(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = object() - self.assertNotEqual(time_range1, time_range2) - - def test___ne__same_value(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - comparison_val = (time_range1 != time_range2) - self.assertFalse(comparison_val) - - def _to_pb_helper(self, start_micros=None, end_micros=None): - import datetime - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._non_upstream_helpers import EPOCH - - pb_kwargs = {} - - start = None - if start_micros is not None: - start = EPOCH + datetime.timedelta(microseconds=start_micros) - pb_kwargs['start_timestamp_micros'] = start_micros - end = None - if end_micros is not None: - end = EPOCH + datetime.timedelta(microseconds=end_micros) - pb_kwargs['end_timestamp_micros'] = end_micros - time_range = self._makeOne(start=start, end=end) - - expected_pb = data_pb2.TimestampRange(**pb_kwargs) - self.assertEqual(time_range.to_pb(), expected_pb) - - def test_to_pb(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - end_micros = 12939371000 - self._to_pb_helper(start_micros=start_micros, - end_micros=end_micros) - - def test_to_pb_start_only(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - self._to_pb_helper(start_micros=start_micros) - - def test_to_pb_end_only(self): - # Makes sure already milliseconds granularity - end_micros = 12939371000 - self._to_pb_helper(end_micros=end_micros) - - -class TestTimestampRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import TimestampRangeFilter - return TimestampRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - range_ = object() - row_filter = self._makeOne(range_) - self.assertTrue(row_filter.range_ is range_) - - def test___eq__type_differ(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - range_ = object() - row_filter1 = self._makeOne(range_) - row_filter2 = self._makeOne(range_) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import TimestampRange - - range_ = TimestampRange() - row_filter = self._makeOne(range_) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - timestamp_range_filter=data_pb2.TimestampRange()) - self.assertEqual(pb_val, expected_pb) - - -class TestValueRegexFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ValueRegexFilter - return ValueRegexFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - regex = b'value-regex' - row_filter = self._makeOne(regex) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_regex_filter=regex) - self.assertEqual(pb_val, expected_pb) - - -class TestValueRangeFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ValueRangeFilter - return ValueRangeFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.start_value, None) - self.assertEqual(row_filter.end_value, None) - self.assertTrue(row_filter.inclusive_start) - self.assertTrue(row_filter.inclusive_end) - - def test_constructor_explicit(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(row_filter.start_value is start_value) - self.assertTrue(row_filter.end_value is end_value) - self.assertTrue(row_filter.inclusive_start is inclusive_start) - self.assertTrue(row_filter.inclusive_end is inclusive_end) - - def test_constructor_bad_start(self): - self.assertRaises(ValueError, self._makeOne, inclusive_start=True) - - def test_constructor_bad_end(self): - self.assertRaises(ValueError, self._makeOne, inclusive_end=True) - - def test___eq__(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - row_filter1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - row_filter2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - row_filter1 = self._makeOne() - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - row_filter = self._makeOne() - expected_pb = data_pb2.RowFilter( - value_range_filter=data_pb2.ValueRange()) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value) - val_range_pb = data_pb2.ValueRange(start_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(start_value=value, inclusive_start=False) - val_range_pb = data_pb2.ValueRange(start_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value) - val_range_pb = data_pb2.ValueRange(end_value_inclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - row_filter = self._makeOne(end_value=value, inclusive_end=False) - val_range_pb = data_pb2.ValueRange(end_value_exclusive=value) - expected_pb = data_pb2.RowFilter(value_range_filter=val_range_pb) - self.assertEqual(row_filter.to_pb(), expected_pb) - - -class Test_CellCountFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import _CellCountFilter - return _CellCountFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - num_cells = object() - row_filter = self._makeOne(num_cells) - self.assertTrue(row_filter.num_cells is num_cells) - - def test___eq__type_differ(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - self.assertEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - num_cells = object() - row_filter1 = self._makeOne(num_cells) - row_filter2 = self._makeOne(num_cells) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - -class TestCellsRowOffsetFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import CellsRowOffsetFilter - return CellsRowOffsetFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 76 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_offset_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsRowLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import CellsRowLimitFilter - return CellsRowLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 189 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(cells_per_row_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestCellsColumnLimitFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import CellsColumnLimitFilter - return CellsColumnLimitFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - num_cells = 10 - row_filter = self._makeOne(num_cells) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter( - cells_per_column_limit_filter=num_cells) - self.assertEqual(pb_val, expected_pb) - - -class TestStripValueTransformerFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import StripValueTransformerFilter - return StripValueTransformerFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - flag = True - row_filter = self._makeOne(flag) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(strip_value_transformer=flag) - self.assertEqual(pb_val, expected_pb) - - -class TestApplyLabelFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ApplyLabelFilter - return ApplyLabelFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - label = object() - row_filter = self._makeOne(label) - self.assertTrue(row_filter.label is label) - - def test___eq__type_differ(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___eq__same_value(self): - label = object() - row_filter1 = self._makeOne(label) - row_filter2 = self._makeOne(label) - self.assertEqual(row_filter1, row_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - label = u'label' - row_filter = self._makeOne(label) - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(apply_label_transformer=label) - self.assertEqual(pb_val, expected_pb) - - -class Test_FilterCombination(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import _FilterCombination - return _FilterCombination - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - row_filter = self._makeOne() - self.assertEqual(row_filter.filters, []) - - def test_constructor_explicit(self): - filters = object() - row_filter = self._makeOne(filters=filters) - self.assertTrue(row_filter.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - -class TestRowFilterChain(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowFilterChain - return RowFilterChain - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import CellsRowLimitFilter - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestRowFilterUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowFilterUnion - return RowFilterUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import CellsRowLimitFilter - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter2 = RowSampleFilter(0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = CellsRowLimitFilter(11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestConditionalRowFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ConditionalRowFilter - return ConditionalRowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertTrue(cond_filter.base_filter is base_filter) - self.assertTrue(cond_filter.true_filter is true_filter) - self.assertTrue(cond_filter.false_filter is false_filter) - - def test___eq__(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertEqual(cond_filter1, cond_filter2) - - def test___eq__type_differ(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = object() - self.assertNotEqual(cond_filter1, cond_filter2) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import CellsRowOffsetFilter - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = CellsRowOffsetFilter(11) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, - false_filter=row_filter3) - filter_pb = row_filter4.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - false_filter=row_filter3_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_true_only(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_false_only(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowSampleFilter - from gcloud_bigtable.row import StripValueTransformerFilter - - row_filter1 = StripValueTransformerFilter(True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowSampleFilter(0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - false_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class Test__parse_family_pb(unittest2.TestCase): - - def _callFUT(self, family_pb): - from gcloud_bigtable.row import _parse_family_pb - return _parse_family_pb(family_pb) - - def test_it(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._non_upstream_helpers import ( - _microseconds_to_timestamp) - - COL_FAM1 = u'col-fam-id' - COL_NAME1 = b'col-name1' - COL_NAME2 = b'col-name2' - CELL_VAL1 = b'cell-val' - CELL_VAL2 = b'cell-val-newer' - CELL_VAL3 = b'altcol-cell-val' - - microseconds = 5554441037 - timestamp = _microseconds_to_timestamp(microseconds) - expected_dict = { - COL_NAME1: [ - (CELL_VAL1, timestamp), - (CELL_VAL2, timestamp), - ], - COL_NAME2: [ - (CELL_VAL3, timestamp), - ], - } - expected_output = (COL_FAM1, expected_dict) - sample_input = data_pb2.Family( - name=COL_FAM1, - columns=[ - data_pb2.Column( - qualifier=COL_NAME1, - cells=[ - data_pb2.Cell( - value=CELL_VAL1, - timestamp_micros=microseconds, - ), - data_pb2.Cell( - value=CELL_VAL2, - timestamp_micros=microseconds, - ), - ], - ), - data_pb2.Column( - qualifier=COL_NAME2, - cells=[ - data_pb2.Cell( - value=CELL_VAL3, - timestamp_micros=microseconds, - ), - ], - ), - ], - ) - self.assertEqual(expected_output, self._callFUT(sample_input)) - - -class _Client(object): - - data_stub = None - - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds - - -class _Cluster(object): - - def __init__(self, client=None): - self._client = client - - -class _Table(object): - - def __init__(self, name, client=None): - self.name = name - self._cluster = _Cluster(client) diff --git a/gcloud_bigtable/test_row_data.py b/gcloud_bigtable/test_row_data.py deleted file mode 100644 index ff86ee9..0000000 --- a/gcloud_bigtable/test_row_data.py +++ /dev/null @@ -1,527 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestCell(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import Cell - return Cell - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _from_pb_test_helper(self, labels=None): - import datetime - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._non_upstream_helpers import EPOCH - - timestamp_micros = 18738724000 # Make sure millis granularity - timestamp = EPOCH + datetime.timedelta(microseconds=timestamp_micros) - value = b'value-bytes' - - if labels is None: - cell_pb = data_pb2.Cell(value=value, - timestamp_micros=timestamp_micros) - cell_expected = self._makeOne(value, timestamp) - else: - cell_pb = data_pb2.Cell(value=value, - timestamp_micros=timestamp_micros, - labels=labels) - cell_expected = self._makeOne(value, timestamp, labels=labels) - - klass = self._getTargetClass() - result = klass.from_pb(cell_pb) - self.assertEqual(result, cell_expected) - - def test_from_pb(self): - self._from_pb_test_helper() - - def test_from_pb_with_labels(self): - labels = [u'label1', u'label2'] - self._from_pb_test_helper(labels) - - def test_constructor(self): - value = object() - timestamp = object() - cell = self._makeOne(value, timestamp) - self.assertEqual(cell.value, value) - self.assertEqual(cell.timestamp, timestamp) - - def test___eq__(self): - value = object() - timestamp = object() - cell1 = self._makeOne(value, timestamp) - cell2 = self._makeOne(value, timestamp) - self.assertEqual(cell1, cell2) - - def test___eq__type_differ(self): - cell1 = self._makeOne(None, None) - cell2 = object() - self.assertNotEqual(cell1, cell2) - - def test___ne__same_value(self): - value = object() - timestamp = object() - cell1 = self._makeOne(value, timestamp) - cell2 = self._makeOne(value, timestamp) - comparison_val = (cell1 != cell2) - self.assertFalse(comparison_val) - - def test___ne__(self): - value1 = 'value1' - value2 = 'value2' - timestamp = object() - cell1 = self._makeOne(value1, timestamp) - cell2 = self._makeOne(value2, timestamp) - self.assertNotEqual(cell1, cell2) - - -class TestPartialRowData(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import PartialRowData - return PartialRowData - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - row_key = object() - partial_row_data = self._makeOne(row_key) - self.assertTrue(partial_row_data._row_key is row_key) - self.assertEqual(partial_row_data._cells, {}) - self.assertFalse(partial_row_data._committed) - self.assertFalse(partial_row_data._chunks_encountered) - - def test___eq__(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data2 = self._makeOne(row_key) - self.assertEqual(partial_row_data1, partial_row_data2) - - def test___eq__type_differ(self): - partial_row_data1 = self._makeOne(None) - partial_row_data2 = object() - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__same_value(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data2 = self._makeOne(row_key) - comparison_val = (partial_row_data1 != partial_row_data2) - self.assertFalse(comparison_val) - - def test___ne__(self): - row_key1 = object() - partial_row_data1 = self._makeOne(row_key1) - row_key2 = object() - partial_row_data2 = self._makeOne(row_key2) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__committed(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data1._committed = object() - partial_row_data2 = self._makeOne(row_key) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__cells(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data1._cells = object() - partial_row_data2 = self._makeOne(row_key) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test_to_dict(self): - cell1 = object() - cell2 = object() - cell3 = object() - - family_name1 = u'name1' - family_name2 = u'name2' - qual1 = b'col1' - qual2 = b'col2' - qual3 = b'col3' - - partial_row_data = self._makeOne(None) - partial_row_data._cells = { - family_name1: { - qual1: cell1, - qual2: cell2, - }, - family_name2: { - qual3: cell3, - }, - } - - result = partial_row_data.to_dict() - col1 = family_name1.encode('ascii') + b':' + qual1 - col2 = family_name1.encode('ascii') + b':' + qual2 - col3 = family_name2.encode('ascii') + b':' + qual3 - expected_result = { - col1: cell1, - col2: cell2, - col3: cell3, - } - self.assertEqual(result, expected_result) - - def test_cells_property(self): - partial_row_data = self._makeOne(None) - cells = {1: 2} - partial_row_data._cells = cells - # Make sure we get a copy, not the original. - self.assertFalse(partial_row_data.cells is cells) - self.assertEqual(partial_row_data.cells, cells) - - def test_row_key_getter(self): - row_key = object() - partial_row_data = self._makeOne(row_key) - self.assertTrue(partial_row_data.row_key is row_key) - - def test_committed_getter(self): - partial_row_data = self._makeOne(None) - partial_row_data._committed = value = object() - self.assertTrue(partial_row_data.committed is value) - - def test_clear(self): - partial_row_data = self._makeOne(None) - cells = {1: 2} - partial_row_data._cells = cells - self.assertEqual(partial_row_data.cells, cells) - partial_row_data._committed = True - partial_row_data._chunks_encountered = True - partial_row_data.clear() - self.assertFalse(partial_row_data.committed) - self.assertFalse(partial_row_data._chunks_encountered) - self.assertEqual(partial_row_data.cells, {}) - - def test__handle_commit_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - index = last_chunk_index = 1 - self.assertFalse(partial_row_data.committed) - partial_row_data._handle_commit_row(chunk, index, last_chunk_index) - self.assertTrue(partial_row_data.committed) - - def test__handle_commit_row_false(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=False) - - with self.assertRaises(ValueError): - partial_row_data._handle_commit_row(chunk, None, None) - - def test__handle_commit_row_not_last_chunk(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - with self.assertRaises(ValueError): - index = 0 - last_chunk_index = 1 - self.assertNotEqual(index, last_chunk_index) - partial_row_data._handle_commit_row(chunk, index, last_chunk_index) - - def test__handle_reset_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - - # Modify the PartialRowData object so we can check it's been cleared. - partial_row_data._cells = {1: 2} - partial_row_data._committed = True - partial_row_data._handle_reset_row(chunk) - self.assertEqual(partial_row_data.cells, {}) - self.assertFalse(partial_row_data.committed) - - def test__handle_reset_row_failure(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=False) - - with self.assertRaises(ValueError): - partial_row_data._handle_reset_row(chunk) - - def test__handle_row_contents(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import Cell - - partial_row_data = self._makeOne(None) - cell1_pb = data_pb2.Cell(timestamp_micros=1, value=b'val1') - cell2_pb = data_pb2.Cell(timestamp_micros=200, value=b'val2') - cell3_pb = data_pb2.Cell(timestamp_micros=300000, value=b'val3') - col1 = b'col1' - col2 = b'col2' - columns = [ - data_pb2.Column(qualifier=col1, cells=[cell1_pb, cell2_pb]), - data_pb2.Column(qualifier=col2, cells=[cell3_pb]), - ] - family_name = u'name' - row_contents = data_pb2.Family(name=family_name, columns=columns) - chunk = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - self.assertEqual(partial_row_data.cells, {}) - partial_row_data._handle_row_contents(chunk) - expected_cells = { - family_name: { - col1: [Cell.from_pb(cell1_pb), Cell.from_pb(cell2_pb)], - col2: [Cell.from_pb(cell3_pb)], - } - } - self.assertEqual(partial_row_data.cells, expected_cells) - - def test_update_from_read_rows(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key = b'row-key' - partial_row_data = self._makeOne(row_key) - - # Set-up chunk1, some data that will be reset by chunk2. - ignored_family_name = u'ignore-name' - row_contents = data_pb2.Family(name=ignored_family_name) - chunk1 = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - # Set-up chunk2, a reset row. - chunk2 = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - - # Set-up chunk3, a column family with no columns. - family_name = u'name' - row_contents = data_pb2.Family(name=family_name) - chunk3 = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - # Set-up chunk4, a commit row. - chunk4 = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - # Prepare request and make sure PartialRowData is empty before. - read_rows_response_pb = messages_pb2.ReadRowsResponse( - row_key=row_key, chunks=[chunk1, chunk2, chunk3, chunk4]) - self.assertEqual(partial_row_data.cells, {}) - self.assertFalse(partial_row_data.committed) - self.assertFalse(partial_row_data._chunks_encountered) - - # Parse the response and make sure the cells took place. - partial_row_data.update_from_read_rows(read_rows_response_pb) - self.assertEqual(partial_row_data.cells, {family_name: {}}) - self.assertFalse(ignored_family_name in partial_row_data.cells) - self.assertTrue(partial_row_data.committed) - self.assertTrue(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_while_committed(self): - partial_row_data = self._makeOne(None) - partial_row_data._committed = True - self.assertFalse(partial_row_data._chunks_encountered) - - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(None) - - self.assertFalse(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_row_key_disagree(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key1 = b'row-key1' - row_key2 = b'row-key2' - partial_row_data = self._makeOne(row_key1) - self.assertFalse(partial_row_data._chunks_encountered) - - self.assertNotEqual(row_key1, row_key2) - read_rows_response_pb = messages_pb2.ReadRowsResponse(row_key=row_key2) - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(read_rows_response_pb) - - self.assertFalse(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_empty_chunk(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key = b'row-key' - partial_row_data = self._makeOne(row_key) - self.assertFalse(partial_row_data._chunks_encountered) - - chunk = messages_pb2.ReadRowsResponse.Chunk() - read_rows_response_pb = messages_pb2.ReadRowsResponse( - row_key=row_key, chunks=[chunk]) - - # This makes it an "empty" chunk. - self.assertEqual(chunk.WhichOneof('chunk'), None) - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(read_rows_response_pb) - - self.assertFalse(partial_row_data._chunks_encountered) - - -class TestPartialRowsData(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import PartialRowsData - return PartialRowsData - - def _getDoNothingClass(self): - klass = self._getTargetClass() - - class FakePartialRowsData(klass): - - def __init__(self, *args, **kwargs): - super(FakePartialRowsData, self).__init__(*args, **kwargs) - self._consumed = [] - - def consume_next(self): - value = self._response_iterator.next() - self._consumed.append(value) - return value - - return FakePartialRowsData - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - response_iterator = object() - partial_rows_data = self._makeOne(response_iterator) - self.assertTrue(partial_rows_data._response_iterator - is response_iterator) - self.assertEqual(partial_rows_data._rows, {}) - - def test_rows_getter(self): - partial_rows_data = self._makeOne(None) - partial_rows_data._rows = value = object() - self.assertTrue(partial_rows_data.rows is value) - - def test___eq__(self): - response_iterator = object() - partial_rows_data1 = self._makeOne(response_iterator) - partial_rows_data2 = self._makeOne(response_iterator) - self.assertEqual(partial_rows_data1, partial_rows_data2) - - def test___eq__type_differ(self): - partial_rows_data1 = self._makeOne(None) - partial_rows_data2 = object() - self.assertNotEqual(partial_rows_data1, partial_rows_data2) - - def test___ne__same_value(self): - response_iterator = object() - partial_rows_data1 = self._makeOne(response_iterator) - partial_rows_data2 = self._makeOne(response_iterator) - comparison_val = (partial_rows_data1 != partial_rows_data2) - self.assertFalse(comparison_val) - - def test___ne__(self): - response_iterator1 = object() - partial_rows_data1 = self._makeOne(response_iterator1) - response_iterator2 = object() - partial_rows_data2 = self._makeOne(response_iterator2) - self.assertNotEqual(partial_rows_data1, partial_rows_data2) - - def test_cancel(self): - response_iterator = _MockCancellableIterator() - partial_rows_data = self._makeOne(response_iterator) - self.assertEqual(response_iterator.cancel_calls, 0) - partial_rows_data.cancel() - self.assertEqual(response_iterator.cancel_calls, 1) - - def test_consume_next(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import PartialRowData - - row_key = b'row-key' - value_pb = messages_pb2.ReadRowsResponse(row_key=row_key) - response_iterator = _MockCancellableIterator(value_pb) - partial_rows_data = self._makeOne(response_iterator) - self.assertEqual(partial_rows_data.rows, {}) - partial_rows_data.consume_next() - expected_rows = {row_key: PartialRowData(row_key)} - self.assertEqual(partial_rows_data.rows, expected_rows) - - def test_consume_next_row_exists(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import PartialRowData - - row_key = b'row-key' - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - value_pb = messages_pb2.ReadRowsResponse(row_key=row_key, - chunks=[chunk]) - response_iterator = _MockCancellableIterator(value_pb) - partial_rows_data = self._makeOne(response_iterator) - existing_values = PartialRowData(row_key) - partial_rows_data._rows[row_key] = existing_values - self.assertFalse(existing_values.committed) - partial_rows_data.consume_next() - self.assertTrue(existing_values.committed) - self.assertEqual(existing_values.cells, {}) - - def test_consume_next_empty_iter(self): - response_iterator = _MockCancellableIterator() - partial_rows_data = self._makeOne(response_iterator) - with self.assertRaises(StopIteration): - partial_rows_data.consume_next() - - def test_consume_all(self): - klass = self._getDoNothingClass() - - value1, value2, value3 = object(), object(), object() - response_iterator = _MockCancellableIterator(value1, value2, value3) - partial_rows_data = klass(response_iterator) - self.assertEqual(partial_rows_data._consumed, []) - partial_rows_data.consume_all() - self.assertEqual(partial_rows_data._consumed, [value1, value2, value3]) - - def test_consume_all_with_max_loops(self): - klass = self._getDoNothingClass() - - value1, value2, value3 = object(), object(), object() - response_iterator = _MockCancellableIterator(value1, value2, value3) - partial_rows_data = klass(response_iterator) - self.assertEqual(partial_rows_data._consumed, []) - partial_rows_data.consume_all(max_loops=1) - self.assertEqual(partial_rows_data._consumed, [value1]) - # Make sure the iterator still has the remaining values. - self.assertEqual(list(response_iterator.iter_values), [value2, value3]) - - -class _MockCancellableIterator(object): - - cancel_calls = 0 - - def __init__(self, *values): - self.iter_values = iter(values) - - def cancel(self): - self.cancel_calls += 1 - - def next(self): - return next(self.iter_values) diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py deleted file mode 100644 index e63aac0..0000000 --- a/gcloud_bigtable/test_table.py +++ /dev/null @@ -1,574 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' -TABLE_ID = 'table-id' - - -class TestTable(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.table import Table - return Table - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - cluster = object() - table = self._makeOne(TABLE_ID, cluster) - self.assertEqual(table.table_id, TABLE_ID) - self.assertTrue(table._cluster is cluster) - - def test_name_property(self): - cluster_name = 'cluster_name' - cluster = _Cluster(cluster_name) - table = self._makeOne(TABLE_ID, cluster) - expected_name = cluster_name + '/tables/' + TABLE_ID - self.assertEqual(table.name, expected_name) - - def test_column_family_factory(self): - from gcloud_bigtable.column_family import ColumnFamily - - table = self._makeOne(TABLE_ID, None) - gc_rule = object() - column_family_id = 'column_family_id' - column_family = table.column_family(column_family_id, gc_rule=gc_rule) - self.assertTrue(isinstance(column_family, ColumnFamily)) - self.assertEqual(column_family.column_family_id, column_family_id) - self.assertTrue(column_family.gc_rule is gc_rule) - self.assertEqual(column_family._table, table) - - def test_row_factory(self): - from gcloud_bigtable.row import Row - - table = self._makeOne(TABLE_ID, None) - row_key = b'row_key' - filter_ = object() - row = table.row(row_key, filter_=filter_) - self.assertTrue(isinstance(row, Row)) - self.assertEqual(row._row_key, row_key) - self.assertEqual(row._table, table) - self.assertEqual(row._filter, filter_) - - def test___eq__(self): - table_id = 'table_id' - cluster = object() - table1 = self._makeOne(table_id, cluster) - table2 = self._makeOne(table_id, cluster) - self.assertEqual(table1, table2) - - def test___eq__type_differ(self): - table1 = self._makeOne('table_id', None) - table2 = object() - self.assertNotEqual(table1, table2) - - def test___ne__same_value(self): - table_id = 'table_id' - cluster = object() - table1 = self._makeOne(table_id, cluster) - table2 = self._makeOne(table_id, cluster) - comparison_val = (table1 != table2) - self.assertFalse(comparison_val) - - def test___ne__(self): - table1 = self._makeOne('table_id1', 'cluster1') - table2 = self._makeOne('table_id2', 'cluster2') - self.assertNotEqual(table1, table2) - - def _create_test_helper(self, initial_split_keys): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 150 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = messages_pb2.CreateTableRequest( - initial_split_keys=initial_split_keys, - name=cluster_name, - table_id=TABLE_ID, - ) - - # Create response_pb - response_pb = data_pb2.Table() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # create() has no return value. - - # Perform the method and check the result. - result = table.create(initial_split_keys=initial_split_keys) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_create(self): - initial_split_keys = None - self._create_test_helper(initial_split_keys) - - def test_create_with_split_keys(self): - initial_split_keys = ['s1', 's2'] - self._create_test_helper(initial_split_keys) - - def test_rename(self): - from google.protobuf import empty_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - new_table_id = 'new_table_id' - self.assertNotEqual(new_table_id, TABLE_ID) - - timeout_seconds = 97 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.RenameTableRequest( - name=table_name, - new_id=new_table_id, - ) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # rename() has no return value. - - # Perform the method and check the result. - result = table.rename(new_table_id) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'RenameTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_delete(self): - from google.protobuf import empty_pb2 - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 871 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.DeleteTableRequest(name=table_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - result = table.delete() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteTable', - (request_pb, timeout_seconds), - {}, - )]) - - def _list_column_families_helper(self, column_family_name=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.column_family import ColumnFamily - - timeout_seconds = 502 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.GetTableRequest(name=table_name) - - # Create response_pb - column_family_id = 'foo' - if column_family_name is None: - column_family_name = (table_name + '/columnFamilies/' + - column_family_id) - column_family = data_pb2.ColumnFamily(name=column_family_name) - response_pb = data_pb2.Table( - column_families={column_family_id: column_family}, - ) - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = { - column_family_id: ColumnFamily(column_family_id, table), - } - - # Perform the method and check the result. - result = table.list_column_families() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_column_families(self): - self._list_column_families_helper() - - def test_list_column_families_failure(self): - column_family_name = 'not-the-right-format' - with self.assertRaises(ValueError): - self._list_column_families_helper( - column_family_name=column_family_name) - - def _read_row_helper(self, chunks): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.row_data import PartialRowData - from gcloud_bigtable import table as MUT - - timeout_seconds = 596 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = object() # Returned by our mock. - mock_create_row_request = _MockCalled(request_pb) - - # Create response_iterator - row_key = b'row-key' - response_pb = messages_pb2.ReadRowsResponse(row_key=row_key, - chunks=chunks) - response_iterator = [response_pb] - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - if chunks: - expected_result = PartialRowData(row_key) - expected_result._committed = True - expected_result._chunks_encountered = True - else: - expected_result = None - - # Perform the method and check the result. - filter_obj = object() - with _Monkey(MUT, _create_row_request=mock_create_row_request): - result = table.read_row(row_key, filter_=filter_obj) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadRows', - (request_pb, timeout_seconds), - {}, - )]) - mock_create_row_request.check_called( - self, [(table.name,)], - [{'row_key': row_key, 'filter_': filter_obj}]) - - def test_read_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - chunks = [chunk] - self._read_row_helper(chunks) - - def test_read_empty_row(self): - chunks = [] - self._read_row_helper(chunks) - - def test_read_row_still_partial(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - # There is never a "commit row". - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - chunks = [chunk] - with self.assertRaises(ValueError): - self._read_row_helper(chunks) - - def test_read_rows(self): - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.row_data import PartialRowsData - from gcloud_bigtable import table as MUT - - timeout_seconds = 1111 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = object() # Returned by our mock. - mock_create_row_request = _MockCalled(request_pb) - - # Create response_iterator - response_iterator = object() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - expected_result = PartialRowsData(response_iterator) - - # Perform the method and check the result. - start_key = b'start-key' - end_key = b'end-key' - filter_obj = object() - allow_row_interleaving = True - limit = 22 - with _Monkey(MUT, _create_row_request=mock_create_row_request): - result = table.read_rows( - start_key=start_key, end_key=end_key, filter_=filter_obj, - allow_row_interleaving=allow_row_interleaving, limit=limit) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadRows', - (request_pb, timeout_seconds), - {}, - )]) - created_kwargs = { - 'start_key': start_key, - 'end_key': end_key, - 'filter_': filter_obj, - 'allow_row_interleaving': allow_row_interleaving, - 'limit': limit, - } - mock_create_row_request.check_called(self, [(table.name,)], - [created_kwargs]) - - def test_sample_row_keys(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - timeout_seconds = 1333 - client = _Client(timeout_seconds=timeout_seconds) - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.SampleRowKeysRequest(table_name=table_name) - - # Create response_iterator - response_iterator = object() # Just passed to a mock. - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - expected_result = response_iterator - - # Perform the method and check the result. - result = table.sample_row_keys() - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'SampleRowKeys', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test__create_row_request(unittest2.TestCase): - - def _callFUT(self, table_name, row_key=None, start_key=None, end_key=None, - filter_=None, allow_row_interleaving=None, limit=None): - from gcloud_bigtable.table import _create_row_request - return _create_row_request( - table_name, row_key=row_key, start_key=start_key, end_key=end_key, - filter_=filter_, allow_row_interleaving=allow_row_interleaving, - limit=limit) - - def test_table_name_only(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - result = self._callFUT(table_name) - expected_result = messages_pb2.ReadRowsRequest(table_name=table_name) - self.assertEqual(result, expected_result) - - def test_row_key_row_range_conflict(self): - with self.assertRaises(ValueError): - self._callFUT(None, row_key=object(), end_key=object()) - - def test_row_key(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - row_key = b'row_key' - result = self._callFUT(table_name, row_key=row_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_key=row_key, - ) - self.assertEqual(result, expected_result) - - def test_row_range_start_key(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - start_key = b'start_key' - result = self._callFUT(table_name, start_key=start_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(start_key=start_key), - ) - self.assertEqual(result, expected_result) - - def test_row_range_end_key(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - end_key = b'end_key' - result = self._callFUT(table_name, end_key=end_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(end_key=end_key), - ) - self.assertEqual(result, expected_result) - - def test_row_range_both_keys(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - start_key = b'start_key' - end_key = b'end_key' - result = self._callFUT(table_name, start_key=start_key, - end_key=end_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(start_key=start_key, end_key=end_key), - ) - self.assertEqual(result, expected_result) - - def test_with_filter(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row import RowSampleFilter - - table_name = 'table_name' - row_filter = RowSampleFilter(0.33) - result = self._callFUT(table_name, filter_=row_filter) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - filter=row_filter.to_pb(), - ) - self.assertEqual(result, expected_result) - - def test_with_allow_row_interleaving(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - allow_row_interleaving = True - result = self._callFUT(table_name, - allow_row_interleaving=allow_row_interleaving) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - allow_row_interleaving=allow_row_interleaving, - ) - self.assertEqual(result, expected_result) - - def test_with_limit(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - limit = 1337 - result = self._callFUT(table_name, limit=limit) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - num_rows_limit=limit, - ) - self.assertEqual(result, expected_result) - - -class _Client(object): - - data_stub = None - cluster_stub = None - operations_stub = None - table_stub = None - - def __init__(self, timeout_seconds=None): - self.timeout_seconds = timeout_seconds - - -class _Cluster(object): - - def __init__(self, name, client=None): - self.name = name - self._client = client diff --git a/pylintrc_default b/pylintrc_default deleted file mode 100644 index 6fe26e9..0000000 --- a/pylintrc_default +++ /dev/null @@ -1,416 +0,0 @@ -# PyLint config for 'gcloud' *library* code. -# -# NOTES: -# -# - Rules for test / demo code are generated into 'pylintrc_reduced' -# as deltas from this configuration by the 'run_pylint.py' script. -# -# - 'RATIONALE: API mapping' as a defense for non-default settings is -# based on the fact that this library maps APIs which are outside our -# control, and adhereing to the out-of-the-box defaults would induce -# breakage / complexity in those mappings -# -[MASTER] - -# Specify a configuration file. -# DEFAULT: rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -# DEFAULT: init-hook= - -# Profiled execution. -# DEFAULT: profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -# DEFAULT: ignore=CVS -# NOTE: This path must be relative due to the use of -# os.walk in astroid.modutils.get_module_files. -# RATIONALE: -# *_pb2.py: protobuf-generated code. -ignore = - bigtable_cluster_data_pb2.py, - bigtable_cluster_service_messages_pb2.py, - bigtable_cluster_service_pb2.py, - bigtable_data_pb2.py, - bigtable_service_messages_pb2.py, - bigtable_service_pb2.py, - bigtable_table_data_pb2.py, - bigtable_table_service_messages_pb2.py, - bigtable_table_service_pb2.py, - -# Pickle collected data for later comparisons. -# DEFAULT: persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -# DEFAULT: load-plugins= -# RATIONALE: We want to make sure our docstrings match the objects -# they document. -load-plugins=pylint.extensions.check_docs - -# DEPRECATED -# DEFAULT: include-ids=no - -# DEPRECATED -# DEFAULT: symbols=no - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -# DEFAULT: enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# DEFAULT: disable= -# RATIONALE: -# - maybe-no-member: bi-modal functions confuse pylint type inference. -# - no-member: indirections in protobuf-generated code -# - protected-access: helpers use '_foo' of classes from generated code. -# - redefined-builtin: use of 'id', 'type', 'filter' args in API-bound funcs; -# use of 'NotImplemented' to map HTTP response code. -# - similarities: 'Bucket' and 'Blob' define 'metageneration' and 'owner' with -# identical implementation but different docstrings. -# - star-args: standard Python idioms for varargs: -# ancestor = Query().filter(*order_props) -# - method-hidden: Decorating a method in a class (e.g. in _DefaultsContainer) -# @_lazy_property_deco -# def dataset_id(): -# ... -# - redefined-variable-type: This error is overzealous and complains at e.g. -# if some_prop: -# return int(value) -# else: -# return float(value) -disable = - maybe-no-member, - no-member, - protected-access, - redefined-builtin, - similarities, - star-args, - method-hidden, - redefined-variable-type, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -# DEFAULT: output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -# DEFAULT: files-output=no - -# Tells whether to display a full report or only the messages -# DEFAULT: reports=yes -# RATIONALE: run from Travis / tox, and don't need / want to parse output. -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -# DEFAULT: evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -# DEFAULT: comment=no - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -# DEFAULT: min-similarity-lines=4 - -# Ignore comments when computing similarities. -# DEFAULT: ignore-comments=yes - -# Ignore docstrings when computing similarities. -# DEFAULT: ignore-docstrings=yes - -# Ignore imports when computing similarities. -# DEFAULT: ignore-imports=no - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -# DEFAULT: init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -# DEFAULT: dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -# DEFAULT: additional-builtins= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -# DEFAULT: logging-modules=logging - - -[FORMAT] - -# Maximum number of characters on a single line. -# DEFAULT: max-line-length=80 - -# Regexp for a line that is allowed to be longer than the limit. -# DEFAULT: ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -# DEFAULT: single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled -# DEFAULT: no-space-check=trailing-comma,dict-separator -# RATIONALE: pylint ignores whitespace checks around the -# constructs "dict-separator" (cases like {1:2}) and -# "trailing-comma" (cases like {1: 2, }). -# By setting "no-space-check" to empty whitespace checks will be -# enforced around both constructs. -no-space-check = - -# Maximum number of lines in a module -# DEFAULT: max-module-lines=1000 -# RATIONALE: API-mapping -max-module-lines=1764 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -# DEFAULT: indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -# DEFAULT: indent-after-paren=4 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -# DEFAULT: notes=FIXME,XXX,TODO - - -[BASIC] - -# Required attributes for module, separated by a comma -# DEFAULT: required-attributes= - -# List of builtins function names that should not be used, separated by a comma -# DEFAULT: bad-functions=map,filter,apply,input,file - -# Good variable names which should always be accepted, separated by a comma -# DEFAULT: good-names=i,j,k,ex,Run,_ -# RATIONALE: 'pb' and 'id' have well-understood meainings in the code. -good-names = i, j, k, ex, Run, _, - pb, - id, - -# Bad variable names which should always be refused, separated by a comma -# DEFAULT: bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -# DEFAULT: name-group= - -# Include a hint for the correct naming format with invalid-name -# DEFAULT: include-naming-hint=no - -# Regular expression matching correct function names -# DEFAULT: function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -# DEFAULT: function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -# DEFAULT: variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -# DEFAULT: variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -# DEFAULT: const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -# DEFAULT: const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -# DEFAULT: attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -# DEFAULT: attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -# DEFAULT: argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -# DEFAULT: argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -# DEFAULT: class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -# DEFAULT: class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -# DEFAULT: inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -# DEFAULT: inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -# DEFAULT: class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -# DEFAULT: class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -# DEFAULT: module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -# DEFAULT: module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -# DEFAULT: method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -# DEFAULT: method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -# DEFAULT: no-docstring-rgx=__.*__ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -# DEFAULT: docstring-min-length=-1 - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -# DEFAULT: ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis -# DEFAULT: ignored-modules= - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -# DEFAULT: ignored-classes=SQLObject - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -# DEFAULT: zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -# DEFAULT: generated-members=REQUEST,acl_users,aq_parent - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -# DEFAULT: deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -# DEFAULT: import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -# DEFAULT: ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -# DEFAULT: int-import-graph= - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -# DEFAULT: ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -# DEFAULT: defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -# DEFAULT: valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -# DEFAULT: valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -# DEFAULT: max-args=5 -# RATIONALE: RowFilter constructor -max-args = 13 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -# DEFAULT: ignored-argument-names=_.* - -# Maximum number of locals for function / method body -# DEFAULT: max-locals=15 -max-locals=20 - -# Maximum number of return / yield for function / method body -# DEFAULT: max-returns=6 - -# Maximum number of branch for function / method body -# DEFAULT: max-branches=12 - -# Maximum number of statements in function / method body -# DEFAULT: max-statements=50 - -# Maximum number of parents for a class (see R0901). -# DEFAULT: max-parents=7 - -# Maximum number of attributes for a class (see R0902). -# DEFAULT: max-attributes=7 -# RATIONALE: API mapping -max-attributes=15 - -# Minimum number of public methods for a class (see R0903). -# DEFAULT: min-public-methods=2 -# RATIONALE: context mgrs may have *no* public methods -min-public-methods=0 - -# Maximum number of public methods for a class (see R0904). -# DEFAULT: max-public-methods=20 -# RATIONALE: API mapping -max-public-methods=40 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -# DEFAULT: overgeneral-exceptions=Exception diff --git a/run_pylint.py b/run_pylint.py deleted file mode 100644 index 5dbea3a..0000000 --- a/run_pylint.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Custom script to run PyLint on gcloud-python-bigtable codebase. - -This runs pylint as a script via subprocess in two different -subprocesses. The first lints the production/library code -using the default rc file (PRODUCTION_RC). The second lints the -demo/test code using an rc file (TEST_RC) which allows more style -violations (hence it has a reduced number of style checks). -""" - -import ConfigParser -import copy -import os -import subprocess -import sys - - -IGNORED_DIRECTORIES = [ - 'gcloud_bigtable/_generated', -] -IGNORED_FILES = [ - 'docs/conf.py', - 'setup.py', -] -PRODUCTION_RC = 'pylintrc_default' -TEST_RC = 'pylintrc_reduced' -TEST_DISABLED_MESSAGES = [ - 'attribute-defined-outside-init', - 'exec-used', - 'import-error', - 'invalid-name', - 'missing-docstring', - 'no-init', - 'no-self-use', - 'superfluous-parens', - 'too-few-public-methods', - 'too-many-locals', - 'too-many-public-methods', - 'unbalanced-tuple-unpacking', - 'abstract-method', -] -TEST_RC_ADDITIONS = { - 'MESSAGES CONTROL': { - 'disable': ', '.join(TEST_DISABLED_MESSAGES), - }, -} - - -def read_config(filename): - """Reads pylintrc config onto native ConfigParser object.""" - config = ConfigParser.ConfigParser() - with open(filename, 'r') as file_obj: - config.readfp(file_obj) - return config - - -def make_test_rc(base_rc_filename, additions_dict, target_filename): - """Combines a base rc and test additions into single file.""" - main_cfg = read_config(base_rc_filename) - - # Create fresh config for test, which must extend production. - test_cfg = ConfigParser.ConfigParser() - test_cfg._sections = copy.deepcopy(main_cfg._sections) - - for section, opts in additions_dict.items(): - curr_section = test_cfg._sections.setdefault( - section, test_cfg._dict()) - for opt, opt_val in opts.items(): - curr_val = curr_section.get(opt) - if curr_val is None: - raise KeyError('Expected to be adding to existing option.') - curr_val = curr_val.rstrip(',') - curr_section[opt] = '%s, %s' % (curr_val, opt_val) - - with open(target_filename, 'w') as file_obj: - test_cfg.write(file_obj) - - -def valid_filename(filename): - """Checks if a file is a Python file and is not ignored.""" - for directory in IGNORED_DIRECTORIES: - if filename.startswith(directory): - return False - return (filename.endswith('.py') and - filename not in IGNORED_FILES) - - -def is_production_filename(filename): - """Checks if the file contains production code. - - :rtype: bool - :returns: Boolean indicating production status. - """ - return not ('demo' in filename or 'test' in filename) - - -def get_files_for_linting(allow_limited=True): - """Gets a list of files in the repository. - - By default, returns all files via ``git ls-files``. However, in some cases - uses a specific commit or branch (a so-called diff base) to compare - against for changed files. (This requires ``allow_limited=True``.) - - To speed up linting on Travis pull requests against master, we manually - set the diff base to origin/master. We don't do this on non-pull requests - since origin/master will be equivalent to the currently checked out code. - One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but - this value is not dependable. - - To allow faster local ``tox`` runs, the environment variables - ``GCLOUD_REMOTE_FOR_LINT`` and ``GCLOUD_BRANCH_FOR_LINT`` can be set to - specify a remote branch to diff against. - - :type allow_limited: bool - :param allow_limited: Boolean indicating if a reduced set of files can - be used. - - :rtype: pair - :returns: Tuple of the diff base using the the list of filenames to be - linted. - """ - diff_base = None - if (os.getenv('TRAVIS_BRANCH') == 'master' and - os.getenv('TRAVIS_PULL_REQUEST') != 'false'): - # In the case of a pull request into master, we want to - # diff against HEAD in master. - diff_base = 'origin/master' - elif os.getenv('TRAVIS') is None: - # Only allow specified remote and branch in local dev. - remote = os.getenv('GCLOUD_REMOTE_FOR_LINT') - branch = os.getenv('GCLOUD_BRANCH_FOR_LINT') - if remote is not None and branch is not None: - diff_base = '%s/%s' % (remote, branch) - - if diff_base is not None and allow_limited: - result = subprocess.check_output(['git', 'diff', '--name-only', - diff_base]) - print 'Using files changed relative to %s:' % (diff_base,) - print '-' * 60 - print result.rstrip('\n') # Don't print trailing newlines. - print '-' * 60 - else: - print 'Diff base not specified, listing all files in repository.' - result = subprocess.check_output(['git', 'ls-files']) - - return result.rstrip('\n').split('\n'), diff_base - - -def get_python_files(all_files=None): - """Gets a list of all Python files in the repository that need linting. - - Relies on :func:`get_files_for_linting()` to determine which files should - be considered. - - NOTE: This requires ``git`` to be installed and requires that this - is run within the ``git`` repository. - - :type all_files: list or :data:`NoneType ` - :param all_files: Optional list of files to be linted. - - :rtype: tuple - :returns: A tuple containing two lists and a boolean. The first list - contains all production files, the next all test/demo files and - the boolean indicates if a restricted fileset was used. - """ - using_restricted = False - if all_files is None: - all_files, diff_base = get_files_for_linting() - using_restricted = diff_base is not None - - library_files = [] - non_library_files = [] - for filename in all_files: - if valid_filename(filename): - if is_production_filename(filename): - library_files.append(filename) - else: - non_library_files.append(filename) - - return library_files, non_library_files, using_restricted - - -def lint_fileset(filenames, rcfile, description): - """Lints a group of files using a given rcfile.""" - # Only lint filenames that exist. For example, 'git diff --name-only' - # could spit out deleted / renamed files. Another alternative could - # be to use 'git diff --name-status' and filter out files with a - # status of 'D'. - filenames = [filename for filename in filenames - if os.path.exists(filename)] - if filenames: - rc_flag = '--rcfile=%s' % (rcfile,) - pylint_shell_command = ['pylint', rc_flag] + filenames - status_code = subprocess.call(pylint_shell_command) - if status_code != 0: - error_message = ('Pylint failed on %s with ' - 'status %d.' % (description, status_code)) - print >> sys.stderr, error_message - sys.exit(status_code) - else: - print 'Skipping %s, no files to lint.' % (description,) - - -def main(): - """Script entry point. Lints both sets of files.""" - make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC) - library_files, non_library_files, using_restricted = get_python_files() - try: - lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') - except SystemExit: - if not using_restricted: - raise - - message = 'Restricted lint failed, expanding to full fileset.' - print >> sys.stderr, message - all_files, _ = get_files_for_linting(allow_limited=False) - library_files, non_library_files, _ = get_python_files( - all_files=all_files) - lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') - - -if __name__ == '__main__': - main() diff --git a/scripts/check_generate.py b/scripts/check_generate.py deleted file mode 100644 index 76c9cd3..0000000 --- a/scripts/check_generate.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Checking that protobuf generated modules import correctly.""" - -from __future__ import print_function - -import glob -import os - - -def main(): - """Import all PB2 files.""" - print('>>> import gcloud_bigtable._generated') - _ = __import__('gcloud_bigtable._generated') - pb2_files = sorted(glob.glob('gcloud_bigtable/_generated/*pb2.py')) - for filename in pb2_files: - basename = os.path.basename(filename) - module_name, _ = os.path.splitext(basename) - - print('>>> from gcloud_bigtable._generated import ' + module_name) - _ = __import__('gcloud_bigtable._generated', fromlist=[module_name]) - - -if __name__ == '__main__': - main() diff --git a/scripts/make_operations_grpc.py b/scripts/make_operations_grpc.py deleted file mode 100644 index d546191..0000000 --- a/scripts/make_operations_grpc.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Get the inserted gRPC lines for datastore pb2 file.""" - -import os -import shutil -import subprocess -import tempfile - - -ROOT_DIR = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..')) -PROTOS_DIR = os.path.join(ROOT_DIR, 'cloud-bigtable-client', - 'bigtable-protos', 'src', 'main', 'proto') -PROTO_PATH = os.path.join(PROTOS_DIR, 'google', 'longrunning', - 'operations.proto') -GRPC_ONLY_FILE = os.path.join(ROOT_DIR, 'gcloud_bigtable', - '_generated', 'operations_grpc_pb2.py') -PROTOC_CMD = 'protoc' -GRPC_PLUGIN = 'grpc_python_plugin' - - -def get_pb2_contents_with_grpc(): - """Get pb2 lines generated by protoc with gRPC plugin. - - :rtype: list - :returns: A list of lines in the generated file. - """ - temp_dir = tempfile.mkdtemp() - generated_path = os.path.join(temp_dir, 'google', 'longrunning', - 'operations_pb2.py') - try: - subprocess.check_output([ - PROTOC_CMD, - '--proto_path', - PROTOS_DIR, - '--python_out', - temp_dir, - '--plugin', - 'protoc-gen-grpc=' + GRPC_PLUGIN, - '--grpc_out', - temp_dir, - PROTO_PATH, - ]) - with open(generated_path, 'rb') as file_obj: - return file_obj.readlines() - finally: - shutil.rmtree(temp_dir, ignore_errors=True) - - -def get_pb2_contents_without_grpc(): - """Get pb2 lines generated by protoc without gRPC plugin. - - :rtype: list - :returns: A list of lines in the generated file. - """ - temp_dir = tempfile.mkdtemp() - generated_path = os.path.join(temp_dir, 'google', 'longrunning', - 'operations_pb2.py') - try: - subprocess.check_output([ - PROTOC_CMD, - '--proto_path', - PROTOS_DIR, - '--python_out', - temp_dir, - PROTO_PATH, - ]) - with open(generated_path, 'rb') as file_obj: - return file_obj.readlines() - finally: - shutil.rmtree(temp_dir, ignore_errors=True) - - -def get_pb2_grpc_only(): - """Get pb2 lines that are only in gRPC. - - :rtype: list - :returns: A list of lines that are only in the pb2 file - generated with the gRPC plugin. - """ - grpc_contents = get_pb2_contents_with_grpc() - non_grpc_contents = get_pb2_contents_without_grpc() - - grpc_only_lines = [] - curr_non_grpc_line = 0 - for line in grpc_contents: - if line == non_grpc_contents[curr_non_grpc_line]: - curr_non_grpc_line += 1 - else: - grpc_only_lines.append(line) - - return grpc_only_lines - - -def main(): - """Write gRPC-only lines to custom module.""" - grpc_only_lines = get_pb2_grpc_only() - with open(GRPC_ONLY_FILE, 'wb') as file_obj: - file_obj.write(''.join(grpc_only_lines)) - - -if __name__ == '__main__': - main() diff --git a/scripts/pep8_on_repo.sh b/scripts/pep8_on_repo.sh deleted file mode 100755 index 6afcd7f..0000000 --- a/scripts/pep8_on_repo.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -pep8 $(git ls-files '*py') diff --git a/scripts/rewrite_imports.py b/scripts/rewrite_imports.py deleted file mode 100644 index 1e5a6e0..0000000 --- a/scripts/rewrite_imports.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Build script for rewriting imports for protobuf generated modules. - -Intended to be used for Google Cloud Bigtable protos (google/bigtable/v1) -and the dependent modules (google/api and google/protobuf). -""" - -import glob - - -IMPORT_TEMPLATE = 'import %s' -IMPORT_FROM_TEMPLATE = 'from %s import ' -REPLACEMENTS = { - 'google.bigtable.admin.cluster.v1': 'gcloud_bigtable._generated', - 'google.bigtable.admin.table.v1': 'gcloud_bigtable._generated', - 'google.bigtable.v1': 'gcloud_bigtable._generated', -} - - -def transform_old_to_new(line, old_module, new_module, - ignore_import_from=False): - """Transforms from an old module to a new one. - - First checks if a line starts with - "from {old_module} import ..." - then checks if the line contains - "import {old_module} ..." - then checks if the line starts with (ignoring whitespace) - "{old_module} ..." - and finally checks if the line contians - "'some-dict-key': {old_module} ..." - - In any of these cases, "{old_module}" is replaced with "{new_module}". - If none match, nothing is returned. - - :type line: str - :param line: The line to be transformed. - - :type old_module: str - :param old_module: The import to be re-written. - - :type new_module: str - :param new_module: The new location of the re-written import. - - :type ignore_import_from: bool - :param ignore_import_from: Flag to determine if the "from * import" - statements should be ignored. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: The transformed line if the old module was found, otherwise - does nothing. - """ - if not ignore_import_from: - import_from_statement = IMPORT_FROM_TEMPLATE % (old_module,) - if line.startswith(import_from_statement): - new_import_from_statement = IMPORT_FROM_TEMPLATE % (new_module,) - # Only replace the first instance of the import statement. - return line.replace(import_from_statement, - new_import_from_statement, 1) - - # If the line doesn't start with a "from * import *" statement, it - # may still contain a "import * ..." statement. - import_statement = IMPORT_TEMPLATE % (old_module,) - if import_statement in line: - new_import_statement = IMPORT_TEMPLATE % (new_module,) - # Only replace the first instance of the import statement. - return line.replace(import_statement, - new_import_statement, 1) - - # Also need to change references to the standalone imports. As a - # stop-gap we fix references to them at the beginning of a line - # (ignoring whitespace). - if line.lstrip().startswith(old_module): - # Only replace the first instance of the old_module. - return line.replace(old_module, new_module, 1) - - # Finally check for usage in dictionaries. - if ': ' + old_module in line: - # Only replace the first instance of the old_module. - return line.replace(': ' + old_module, ': ' + new_module, 1) - - -def transform_line(line): - """Transforms an import line in a PB2 module. - - If the line is not an import of one of the packages in ``REPLACEMENTS``, - does nothing and returns the original. Otherwise it replaces the package - matched with our local package. - - :type line: str - :param line: The line to be transformed. - - :rtype: str - :returns: The transformed line. - """ - for old_module, new_module in REPLACEMENTS.iteritems(): - result = transform_old_to_new(line, old_module, new_module) - if result is not None: - return result - - # If no matches, there is nothing to transform. - return line - - -def rewrite_file(filename): - """Rewrites a given PB2 modules. - - :type filename: str - :param filename: The name of the file to be rewritten. - """ - with open(filename, 'rU') as file_obj: - content_lines = file_obj.read().split('\n') - - new_content = [] - for line in content_lines: - new_content.append(transform_line(line)) - - with open(filename, 'w') as file_obj: - file_obj.write('\n'.join(new_content)) - - -def main(): - """Rewrites all PB2 files.""" - pb2_files = glob.glob('gcloud_bigtable/_generated/*pb2.py') - for filename in pb2_files: - rewrite_file(filename) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py deleted file mode 100644 index 6fe763e..0000000 --- a/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from setuptools import setup -from setuptools import find_packages - - -HERE = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(HERE, 'README.md')) as file_obj: - README = file_obj.read() - - -REQUIREMENTS = [ - 'httplib2 >= 0.9.1', - 'oauth2client >= 1.4.6', - 'protobuf >= 3.0.0b2', - 'pytz', - 'six >= 1.6.1', - 'googleapis-common-protos', -] - -setup( - name='gcloud-bigtable', - version='0.0.1', - description='API Client library for Google Cloud Bigtable', - author='Google Cloud Platform', - author_email='daniel.j.hermes@gmail.com', - long_description=README, - scripts=[], - url='https://github.com/dhermes/gcloud-python-bigtable', - packages=find_packages(), - license='Apache 2.0', - platforms='Posix; MacOS X; Windows', - include_package_data=True, - zip_safe=False, - install_requires=REQUIREMENTS, - classifiers=[ - 'Development Status :: 1 - Planning', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Internet', - ] -) diff --git a/system_tests/local_setup.sample b/system_tests/local_setup.sample deleted file mode 100644 index b8a47a4..0000000 --- a/system_tests/local_setup.sample +++ /dev/null @@ -1,3 +0,0 @@ -export GCLOUD_REMOTE_FOR_LINT="upstream" -export GCLOUD_BRANCH_FOR_LINT="master" -export GCLOUD_TESTS_PROJECT_ID=some-project-id diff --git a/system_tests/run.py b/system_tests/run.py deleted file mode 100644 index c4e7d2e..0000000 --- a/system_tests/run.py +++ /dev/null @@ -1,457 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import datetime -import operator -import os -import time - -import pytz -import unittest2 - -from oauth2client.client import GoogleCredentials - -from gcloud_bigtable._non_upstream_helpers import _microseconds_to_timestamp -from gcloud_bigtable._non_upstream_helpers import _timestamp_to_microseconds -from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import MaxVersionsGCRule -from gcloud_bigtable.row import ApplyLabelFilter -from gcloud_bigtable.row import ColumnQualifierRegexFilter -from gcloud_bigtable.row import RowFilterChain -from gcloud_bigtable.row import RowFilterUnion -from gcloud_bigtable.row_data import Cell -from gcloud_bigtable.row_data import PartialRowData - - -PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID') -CENTRAL_1C_ZONE = 'us-central1-c' -NOW_MILLIS = int(1000 * time.time()) -CLUSTER_ID = 'gcloud-python-%d' % (NOW_MILLIS,) -TABLE_ID = 'gcloud-python-test-table' -COLUMN_FAMILY_ID1 = u'col-fam-id1' -COLUMN_FAMILY_ID2 = u'col-fam-id2' -COL_NAME1 = b'col-name1' -COL_NAME2 = b'col-name2' -COL_NAME3 = b'col-name3-but-other-fam' -CELL_VAL1 = b'cell-val' -CELL_VAL2 = b'cell-val-newer' -CELL_VAL3 = b'altcol-cell-val' -CELL_VAL4 = b'foo' -ROW_KEY = b'row-key' -ROW_KEY_ALT = b'row-key-alt' -EXISTING_CLUSTERS = [] -EXPECTED_ZONES = ( - 'asia-east1-b', - 'europe-west1-c', - 'us-central1-b', - CENTRAL_1C_ZONE, -) -CREDENTIALS = GoogleCredentials.get_application_default() -CLIENT = Client(project=PROJECT_ID, credentials=CREDENTIALS, admin=True) -CLUSTER = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID, - display_name=CLUSTER_ID) - - -def setUpModule(): - CLIENT.start() - clusters, failed_zones = CLIENT.list_clusters() - - if len(failed_zones) != 0: - raise ValueError('List clusters failed in module set up.') - - EXISTING_CLUSTERS[:] = clusters - - # After listing, create the test cluster. - CLUSTER.create() - - -def tearDownModule(): - CLUSTER.delete() - CLIENT.stop() - - -class TestClusterAdminAPI(unittest2.TestCase): - - def setUp(self): - self.clusters_to_delete = [] - - def tearDown(self): - for cluster in self.clusters_to_delete: - cluster.delete() - - def test_list_zones(self): - zones = CLIENT.list_zones() - self.assertEqual(sorted(zones), list(EXPECTED_ZONES)) - - def test_list_clusters(self): - clusters, failed_zones = CLIENT.list_clusters() - self.assertEqual(failed_zones, []) - # We have added one new cluster in `setUpModule`. - self.assertEqual(len(clusters), len(EXISTING_CLUSTERS) + 1) - for cluster in clusters: - cluster_existence = (cluster in EXISTING_CLUSTERS or - cluster == CLUSTER) - self.assertTrue(cluster_existence) - - def test_reload(self): - # Use same arguments as CLUSTER (created in `setUpModule`). - cluster = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID) - # Make sure metadata unset before reloading. - cluster.display_name = None - cluster.serve_nodes = None - - cluster.reload() - self.assertEqual(cluster.display_name, CLUSTER.display_name) - self.assertEqual(cluster.serve_nodes, CLUSTER.serve_nodes) - - def test_create_cluster(self): - cluster_id = '%s-a' % (CLUSTER_ID,) - cluster = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) - operation = cluster.create() - # Make sure this cluster gets deleted after the test case. - self.clusters_to_delete.append(cluster) - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(operation.finished()) - - # Create a new cluster instance and make sure it is the same. - cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) - cluster_alt.reload() - - self.assertEqual(cluster, cluster_alt) - self.assertEqual(cluster.display_name, cluster_alt.display_name) - self.assertEqual(cluster.serve_nodes, cluster_alt.serve_nodes) - - def test_update(self): - curr_display_name = CLUSTER.display_name - CLUSTER.display_name = 'Foo Bar Baz' - operation = CLUSTER.update() - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(operation.finished()) - - # Create a new cluster instance and make sure it is the same. - cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID) - self.assertNotEqual(cluster_alt.display_name, - CLUSTER.display_name) - cluster_alt.reload() - self.assertEqual(cluster_alt.display_name, - CLUSTER.display_name) - - # Make sure to put the cluster back the way it was for the - # other test cases. - CLUSTER.display_name = curr_display_name - operation = CLUSTER.update() - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(operation.finished()) - - -class TestTableAdminAPI(unittest2.TestCase): - - @classmethod - def setUpClass(cls): - cls._table = CLUSTER.table(TABLE_ID) - cls._table.create() - - @classmethod - def tearDownClass(cls): - cls._table.delete() - - def setUp(self): - self.tables_to_delete = [] - - def tearDown(self): - for table in self.tables_to_delete: - table.delete() - - def test_list_tables(self): - # Since `CLUSTER` is newly created in `setUpModule`, the table - # created in `setUpClass` here will be the only one. - tables = CLUSTER.list_tables() - self.assertEqual(tables, [self._table]) - - def test_create_table(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - # First, create a sorted version of our expected result. - name_attr = operator.attrgetter('name') - expected_tables = sorted([temp_table, self._table], key=name_attr) - - # Then query for the tables in the cluster and sort them by - # name as well. - tables = CLUSTER.list_tables() - sorted_tables = sorted(tables, key=name_attr) - self.assertEqual(sorted_tables, expected_tables) - - def test_rename_table(self): - from grpc.beta import interfaces - from grpc.framework.interfaces.face import face - - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - exc_caught = None - try: - temp_table.rename(temp_table_id + '-alt') - except face.LocalError as exc: - exc_caught = exc # Python 3 syntax issue. - self.assertNotEqual(exc_caught, None) - self.assertEqual(exc_caught.code, - interfaces.StatusCode.UNIMPLEMENTED) - self.assertEqual( - exc_caught.details, - 'BigtableTableService.RenameTable is not yet implemented') - - def test_create_column_family(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - self.assertEqual(temp_table.list_column_families(), {}) - gc_rule = MaxVersionsGCRule(1) - column_family = temp_table.column_family(COLUMN_FAMILY_ID1, - gc_rule=gc_rule) - column_family.create() - - col_fams = temp_table.list_column_families() - - self.assertEqual(len(col_fams), 1) - retrieved_col_fam = col_fams[COLUMN_FAMILY_ID1] - self.assertTrue(retrieved_col_fam._table is column_family._table) - self.assertEqual(retrieved_col_fam.column_family_id, - column_family.column_family_id) - self.assertEqual(retrieved_col_fam.gc_rule, gc_rule) - - def test_update_column_family(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - gc_rule = MaxVersionsGCRule(1) - column_family = temp_table.column_family(COLUMN_FAMILY_ID1, - gc_rule=gc_rule) - column_family.create() - - # Check that our created table is as expected. - col_fams = temp_table.list_column_families() - self.assertEqual(col_fams, {COLUMN_FAMILY_ID1: column_family}) - - # Update the column family's GC rule and then try to update. - column_family.gc_rule = None - column_family.update() - - # Check that the update has propagated. - col_fams = temp_table.list_column_families() - self.assertEqual(col_fams[COLUMN_FAMILY_ID1].gc_rule, None) - - def test_delete_column_family(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - self.assertEqual(temp_table.list_column_families(), {}) - column_family = temp_table.column_family(COLUMN_FAMILY_ID1) - column_family.create() - - # Make sure the family is there before deleting it. - col_fams = temp_table.list_column_families() - self.assertEqual(list(col_fams.keys()), [COLUMN_FAMILY_ID1]) - - column_family.delete() - # Make sure we have successfully deleted it. - self.assertEqual(temp_table.list_column_families(), {}) - - -class TestDataAPI(unittest2.TestCase): - - @classmethod - def setUpClass(cls): - cls._table = table = CLUSTER.table(TABLE_ID) - table.create() - table.column_family(COLUMN_FAMILY_ID1).create() - table.column_family(COLUMN_FAMILY_ID2).create() - - @classmethod - def tearDownClass(cls): - # Will also delete any data contained in the table. - cls._table.delete() - - def setUp(self): - self.rows_to_delete = [] - - def tearDown(self): - for row in self.rows_to_delete: - row.clear_mutations() - row.delete() - row.commit() - - def _write_to_row(self, row1=None, row2=None, row3=None, row4=None): - timestamp1 = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - # Must be millisecond granularity. - timestamp1 = _microseconds_to_timestamp( - _timestamp_to_microseconds(timestamp1)) - # 1000 microseconds is a millisecond - timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000) - timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000) - timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000) - if row1 is not None: - row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, - timestamp=timestamp1) - if row2 is not None: - row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, - timestamp=timestamp2) - if row3 is not None: - row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, - timestamp=timestamp3) - if row4 is not None: - row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, - timestamp=timestamp4) - - # Create the cells we will check. - cell1 = Cell(CELL_VAL1, timestamp1) - cell2 = Cell(CELL_VAL2, timestamp2) - cell3 = Cell(CELL_VAL3, timestamp3) - cell4 = Cell(CELL_VAL4, timestamp4) - return cell1, cell2, cell3, cell4 - - def test_read_row(self): - row = self._table.row(ROW_KEY) - self.rows_to_delete.append(row) - - cell1, cell2, cell3, cell4 = self._write_to_row(row, row, row, row) - row.commit() - - # Read back the contents of the row. - partial_row_data = self._table.read_row(ROW_KEY) - self.assertTrue(partial_row_data.committed) - self.assertEqual(partial_row_data.row_key, ROW_KEY) - - # Check the cells match. - ts_attr = operator.attrgetter('timestamp') - expected_row_contents = { - COLUMN_FAMILY_ID1: { - COL_NAME1: sorted([cell1, cell2], key=ts_attr, reverse=True), - COL_NAME2: [cell3], - }, - COLUMN_FAMILY_ID2: { - COL_NAME3: [cell4], - }, - } - self.assertEqual(partial_row_data.cells, expected_row_contents) - - def test_read_rows(self): - row = self._table.row(ROW_KEY) - row_alt = self._table.row(ROW_KEY_ALT) - self.rows_to_delete.extend([row, row_alt]) - - cell1, cell2, cell3, cell4 = self._write_to_row(row, row_alt, - row, row_alt) - row.commit() - row_alt.commit() - - rows_data = self._table.read_rows() - self.assertEqual(rows_data.rows, {}) - rows_data.consume_all() - - # NOTE: We should refrain from editing protected data on instances. - # Instead we should make the values public or provide factories - # for constructing objects with them. - row_data = PartialRowData(ROW_KEY) - row_data._chunks_encountered = True - row_data._committed = True - row_data._cells = { - COLUMN_FAMILY_ID1: { - COL_NAME1: [cell1], - COL_NAME2: [cell3], - }, - } - - row_alt_data = PartialRowData(ROW_KEY_ALT) - row_alt_data._chunks_encountered = True - row_alt_data._committed = True - row_alt_data._cells = { - COLUMN_FAMILY_ID1: { - COL_NAME1: [cell2], - }, - COLUMN_FAMILY_ID2: { - COL_NAME3: [cell4], - }, - } - - expected_rows = { - ROW_KEY: row_data, - ROW_KEY_ALT: row_alt_data, - } - self.assertEqual(rows_data.rows, expected_rows) - - def test_read_with_label_applied(self): - row = self._table.row(ROW_KEY) - self.rows_to_delete.append(row) - - cell1, _, cell3, _ = self._write_to_row(row, None, row) - row.commit() - - # Combine a label with column 1. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - col1_filter = ColumnQualifierRegexFilter(COL_NAME1) - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Combine a label with column 2. - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - col2_filter = ColumnQualifierRegexFilter(COL_NAME2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - partial_row_data = self._table.read_row(ROW_KEY, filter_=row_filter) - self.assertTrue(partial_row_data.committed) - self.assertEqual(partial_row_data.row_key, ROW_KEY) - - cells_returned = partial_row_data.cells - col_fam1 = cells_returned.pop(COLUMN_FAMILY_ID1) - # Make sure COLUMN_FAMILY_ID1 was the only key. - self.assertEqual(len(cells_returned), 0) - - cell1_new, = col_fam1.pop(COL_NAME1) - cell3_new, = col_fam1.pop(COL_NAME2) - # Make sure COL_NAME1 and COL_NAME2 were the only keys. - self.assertEqual(len(col_fam1), 0) - - # Check that cell1 has matching values and gained a label. - self.assertEqual(cell1_new.value, cell1.value) - self.assertEqual(cell1_new.timestamp, cell1.timestamp) - self.assertEqual(cell1.labels, []) - self.assertEqual(cell1_new.labels, [label1]) - - # Check that cell3 has matching values and gained a label. - self.assertEqual(cell3_new.value, cell3.value) - self.assertEqual(cell3_new.timestamp, cell3.timestamp) - self.assertEqual(cell3.labels, []) - self.assertEqual(cell3_new.labels, [label2]) diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py deleted file mode 100644 index 94296ad..0000000 --- a/system_tests/run_happybase.py +++ /dev/null @@ -1,997 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import operator -import struct -import time -import unittest2 - - -_PACK_I64 = struct.Struct('>q').pack -_FIRST_ELT = operator.itemgetter(0) - -USING_HBASE = False -NOW_MILLIS = int(1000 * time.time()) -TABLE_NAME = 'table-name' -ALT_TABLE_NAME = 'other-table' -TTL_FOR_TEST = 3 -COL_FAM1 = 'cf1' -COL_FAM2 = 'cf2' -COL_FAM3 = 'cf3' -FAMILIES = { - COL_FAM1: {'max_versions': 10}, - COL_FAM2: {'max_versions': 1, 'time_to_live': TTL_FOR_TEST}, - COL_FAM3: {}, # use defaults -} -ROW_KEY1 = 'row-key1' -ROW_KEY2 = 'row-key2a' -ROW_KEY3 = 'row-key2b' -COL1 = COL_FAM1 + ':qual1' -COL2 = COL_FAM1 + ':qual2' -COL3 = COL_FAM2 + ':qual1' -COL4 = COL_FAM3 + ':qual3' - - -class Config(object): - """Simple namespace for holding test globals.""" - - connection = None - table = None - delete_cluster = False - - -def set_hbase_connection(): - from happybase import Connection - Config.connection = Connection() - - -def set_cloud_bigtable_connection(): - # pylint: disable=no-name-in-module - from grpc.framework.interfaces.face.face import NetworkError - from gcloud_bigtable import _non_upstream_helpers - from gcloud_bigtable import client as client_mod - from gcloud_bigtable.happybase import Connection - # pylint: enable=no-name-in-module - - _non_upstream_helpers.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' - client = client_mod.Client(admin=True) - zone = 'us-central1-c' - cluster_id = 'gcloud-python' - cluster = client.cluster(zone, cluster_id) - client.start() - try: - cluster.reload() - except NetworkError: - operation = cluster.create() - count = 0 - while not operation.finished(): - count += 1 - if count > 5: - raise NetworkError('Create cluster timed out') - time.sleep(0.5) - # If our test run created a new cluster, then we tell the - # config to clean it up when done. - Config.delete_cluster = True - - Config.connection = Connection(cluster=cluster) - - -def get_connection(): - if Config.connection is None: - if USING_HBASE: - set_hbase_connection() - else: - set_cloud_bigtable_connection() - return Config.connection - - -def get_table(): - if Config.table is None: - connection = get_connection() - Config.table = connection.table(TABLE_NAME) - return Config.table - - -def setUpModule(): - if Config.table is None: - connection = get_connection() - if TABLE_NAME not in connection.tables(): - connection.create_table(TABLE_NAME, FAMILIES) - - -def tearDownModule(): - connection = get_connection() - if not USING_HBASE: - connection.delete_table(TABLE_NAME) - if Config.delete_cluster: - connection._cluster.delete() - connection.close() - - -class TestConnection(unittest2.TestCase): - - def test_create_and_delete_table(self): - connection = get_connection() - - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - connection.create_table(ALT_TABLE_NAME, {COL_FAM1: {}}) - self.assertTrue(ALT_TABLE_NAME in connection.tables()) - if USING_HBASE: - connection.delete_table(ALT_TABLE_NAME, disable=True) - else: - connection.delete_table(ALT_TABLE_NAME) - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - - def test_create_table_failure(self): - connection = get_connection() - - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - empty_families = {} - with self.assertRaises(ValueError): - connection.create_table(ALT_TABLE_NAME, empty_families) - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - - -class BaseTableTest(unittest2.TestCase): - - def setUp(self): - self.rows_to_delete = [] - - def tearDown(self): - table = get_table() - for row_key in self.rows_to_delete: - table.delete(row_key) - - -class TestTable_families(BaseTableTest): - - def test_families(self): - table = get_table() - families = table.families() - - self.assertEqual(set(families.keys()), set(FAMILIES.keys())) - for col_fam, settings in FAMILIES.items(): - retrieved = families[col_fam] - for key, value in settings.items(): - if key == 'time_to_live' and USING_HBASE: - # The Thrift API fails to retrieve the TTL for some reason. - continue - self.assertEqual(retrieved[key], value) - - -class TestTable_row(BaseTableTest): - - def test_row_when_empty(self): - table = get_table() - row1 = table.row(ROW_KEY1) - row2 = table.row(ROW_KEY2) - - self.assertEqual(row1, {}) - self.assertEqual(row2, {}) - - def test_row_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - row1_data = { - COL1: value1, - COL2: value2, - COL3: value3, - COL4: value4, - } - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - # Make sure the vanilla write succeeded. - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - # Pick out specific columns. - row1_diff_fams = table.row(ROW_KEY1, columns=[COL1, COL4]) - self.assertEqual(row1_diff_fams, {COL1: value1, COL4: value4}) - row1_single_col = table.row(ROW_KEY1, columns=[COL3]) - self.assertEqual(row1_single_col, {COL3: value3}) - row1_col_fam = table.row(ROW_KEY1, columns=[COL_FAM1]) - self.assertEqual(row1_col_fam, {COL1: value1, COL2: value2}) - row1_fam_qual_overlap1 = table.row(ROW_KEY1, columns=[COL1, COL_FAM1]) - self.assertEqual(row1_fam_qual_overlap1, {COL1: value1, COL2: value2}) - row1_fam_qual_overlap2 = table.row(ROW_KEY1, columns=[COL_FAM1, COL1]) - if USING_HBASE: - # NOTE: This behavior seems to be "incorrect" but that is how - # HappyBase / HBase works. - self.assertEqual(row1_fam_qual_overlap2, {COL1: value1}) - else: - self.assertEqual(row1_fam_qual_overlap2, - {COL1: value1, COL2: value2}) - row1_multiple_col_fams = table.row(ROW_KEY1, - columns=[COL_FAM1, COL_FAM2]) - self.assertEqual(row1_multiple_col_fams, - {COL1: value1, COL2: value2, COL3: value3}) - - def test_row_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - table.put(ROW_KEY1, {COL3: value3}) - - # Make sure the vanilla write succeeded. - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - ts3 = row1[COL3][1] - - expected_row = { - COL1: (value1, ts1), - COL2: (value2, ts2), - COL3: (value3, ts3), - } - self.assertEqual(row1, expected_row) - - # Make sure the timestamps are (strictly) ascending. - self.assertTrue(ts1 < ts2 < ts3) - - # Use timestamps to retrieve row. - first_two = table.row(ROW_KEY1, timestamp=ts2 + 1, - include_timestamp=True) - self.assertEqual(first_two, { - COL1: (value1, ts1), - COL2: (value2, ts2), - }) - first_one = table.row(ROW_KEY1, timestamp=ts2, - include_timestamp=True) - self.assertEqual(first_one, { - COL1: (value1, ts1), - }) - - -class TestTable_rows(BaseTableTest): - - def test_rows(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - - rows = table.rows([ROW_KEY1, ROW_KEY2]) - rows.sort(key=_FIRST_ELT) - - row1, row2 = rows - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - def test_rows_with_returned_timestamps(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - with table.batch() as batch: - batch.put(ROW_KEY1, row1_data) - batch.put(ROW_KEY2, row2_data) - - rows = table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True) - rows.sort(key=_FIRST_ELT) - - row1, row2 = rows - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - - # Drop the keys now that we have checked. - _, row1 = row1 - _, row2 = row2 - - ts = row1[COL1][1] - # All will have the same timestamp since we used batch. - expected_row1_result = {COL1: (value1, ts), COL2: (value2, ts)} - self.assertEqual(row1, expected_row1_result) - if USING_HBASE: - expected_row2_result = {COL1: (value3, ts)} - else: - # NOTE: Since Cloud Bigtable has no concept of batching, the - # server-side timestamps correspond to separate calls - # to row.commit(). We could circumvent this by manually - # using the local time and storing it on mutations before - # sending. - ts3 = row2[COL1][1] - expected_row2_result = {COL1: (value3, ts3)} - self.assertEqual(row2, expected_row2_result) - - def test_rows_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - - # Filter a single column present in both rows. - rows_col1 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL1]) - rows_col1.sort(key=_FIRST_ELT) - row1, row2 = rows_col1 - self.assertEqual(row1, (ROW_KEY1, {COL1: value1})) - self.assertEqual(row2, (ROW_KEY2, {COL1: value3})) - - # Filter a column not present in one row. - rows_col2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL2]) - self.assertEqual(rows_col2, [(ROW_KEY1, {COL2: value2})]) - - # Filter a column family. - rows_col_fam1 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM1]) - rows_col_fam1.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam1 - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - # Filter a column family with no entries. - rows_col_fam2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM2]) - self.assertEqual(rows_col_fam2, []) - - # Filter a column family that overlaps with a column. - rows_col_fam_overlap1 = table.rows([ROW_KEY1, ROW_KEY2], - columns=[COL1, COL_FAM1]) - rows_col_fam_overlap1.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam_overlap1 - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - # Filter a column family that overlaps with a column (opposite order). - rows_col_fam_overlap2 = table.rows([ROW_KEY1, ROW_KEY2], - columns=[COL_FAM1, COL1]) - rows_col_fam_overlap2.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam_overlap2 - if USING_HBASE: - # NOTE: This behavior seems to be "incorrect" but that is how - # HappyBase / HBase works. - self.assertEqual(row1, (ROW_KEY1, {COL1: value1})) - else: - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - def test_rows_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY2, {COL1: value2}) - table.put(ROW_KEY1, {COL2: value3}) - table.put(ROW_KEY1, {COL4: value4}) - - # Just grab the timestamps - rows = table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - _, row1 = row1 - _, row2 = row2 - ts1 = row1[COL1][1] - ts2 = row2[COL1][1] - ts3 = row1[COL2][1] - ts4 = row1[COL4][1] - - # Make sure the timestamps are (strictly) ascending. - self.assertTrue(ts1 < ts2 < ts3 < ts4) - - # Rows before the third timestamp (assumes exclusive endpoint). - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts3, - include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - self.assertEqual(row1, (ROW_KEY1, {COL1: (value1, ts1)})) - self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) - - # All writes (bump the exclusive endpoint by 1 millisecond). - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4 + 1, - include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - row1_all_data = { - COL1: (value1, ts1), - COL2: (value3, ts3), - COL4: (value4, ts4), - } - self.assertEqual(row1, (ROW_KEY1, row1_all_data)) - self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) - - # First three writes, restricted to column 2. - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4, - columns=[COL2], include_timestamp=True) - self.assertEqual(rows, [(ROW_KEY1, {COL2: (value3, ts3)})]) - - -class TestTable_cells(BaseTableTest): - - def test_cells(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL1: value2}) - table.put(ROW_KEY1, {COL1: value3}) - - # Check with no extra arguments. - all_values = table.cells(ROW_KEY1, COL1) - self.assertEqual(all_values, [value3, value2, value1]) - - # Check the timestamp on all the cells. - all_cells = table.cells(ROW_KEY1, COL1, include_timestamp=True) - self.assertEqual(len(all_cells), 3) - - ts3 = all_cells[0][1] - ts2 = all_cells[1][1] - ts1 = all_cells[2][1] - self.assertEqual(all_cells, - [(value3, ts3), (value2, ts2), (value1, ts1)]) - - # Limit to the two latest cells. - latest_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, - versions=2) - self.assertEqual(latest_two, [(value3, ts3), (value2, ts2)]) - - # Limit to cells before the 2nd timestamp (inclusive). - first_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, - timestamp=ts2 + 1) - self.assertEqual(first_two, [(value2, ts2), (value1, ts1)]) - - # Limit to cells before the 2nd timestamp (exclusive). - first_cell = table.cells(ROW_KEY1, COL1, include_timestamp=True, - timestamp=ts2) - self.assertEqual(first_cell, [(value1, ts1)]) - - -class TestTable_scan(BaseTableTest): - - def test_scan_when_empty(self): - table = get_table() - scan_result = list(table.scan()) - self.assertEqual(scan_result, []) - - def test_scan_single_row(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - scan_result = list(table.scan()) - self.assertEqual(scan_result, [(ROW_KEY1, row1_data)]) - - scan_result_cols = list(table.scan(columns=[COL1])) - self.assertEqual(scan_result_cols, [(ROW_KEY1, {COL1: value1})]) - - scan_result_ts = list(table.scan(include_timestamp=True)) - self.assertEqual(len(scan_result_ts), 1) - only_row = scan_result_ts[0] - self.assertEqual(only_row[0], ROW_KEY1) - row_values = only_row[1] - ts = row_values[COL1][1] - self.assertEqual(row_values, {COL1: (value1, ts), COL2: (value2, ts)}) - - if USING_HBASE: - scan_result_sorted = list(table.scan(sorted_columns=True)) - self.assertEqual(len(scan_result_sorted), 1) - only_row = scan_result_sorted[0] - self.assertEqual(only_row[0], ROW_KEY1) - row1_ordered = row1_data.items() - row1_ordered.sort(key=_FIRST_ELT) - self.assertEqual(only_row[1].items(), row1_ordered) - - def test_scan_filters(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - value5 = 'value5' - value6 = 'value6' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL2: value3, COL3: value4} - row3_data = {COL3: value5, COL4: value6} - - # Need to clean-up row1/2/3 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - self.rows_to_delete.append(ROW_KEY3) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - table.put(ROW_KEY3, row3_data) - - # Basic scan (no filters) - scan_result = list(table.scan()) - self.assertEqual(scan_result, [ - (ROW_KEY1, row1_data), - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # Limit the size of the scan - scan_result = list(table.scan(limit=1)) - self.assertEqual(scan_result, [ - (ROW_KEY1, row1_data), - ]) - - # Scan with a row prefix. - prefix = ROW_KEY2[:-1] - self.assertEqual(prefix, ROW_KEY3[:-1]) - scan_result_prefixed = list(table.scan(row_prefix=prefix)) - self.assertEqual(scan_result_prefixed, [ - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # Make sure our keys are sorted in order - row_keys = [ROW_KEY1, ROW_KEY2, ROW_KEY3] - self.assertEqual(row_keys, sorted(row_keys)) - - # row_start alone (inclusive) - scan_result_row_start = list(table.scan(row_start=ROW_KEY2)) - self.assertEqual(scan_result_row_start, [ - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # row_stop alone (exclusive) - scan_result_row_stop = list(table.scan(row_stop=ROW_KEY2)) - self.assertEqual(scan_result_row_stop, [ - (ROW_KEY1, row1_data), - ]) - - # Both row_start and row_stop - scan_result_row_stop_and_start = list( - table.scan(row_start=ROW_KEY1, row_stop=ROW_KEY3)) - self.assertEqual(scan_result_row_stop_and_start, [ - (ROW_KEY1, row1_data), - (ROW_KEY2, row2_data), - ]) - - if USING_HBASE: - # Using a filter. - scan_result_filter = list(table.scan(filter='KeyOnlyFilter ()')) - self.assertEqual(scan_result_filter, [ - (ROW_KEY1, {COL1: '', COL2: ''}), # Keys only - (ROW_KEY2, {COL2: '', COL3: ''}), # Keys only - (ROW_KEY3, {COL3: '', COL4: ''}), # Keys only - ]) - - def test_scan_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - value5 = 'value5' - value6 = 'value6' - - # Need to clean-up row1/2/3 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - self.rows_to_delete.append(ROW_KEY3) - table.put(ROW_KEY3, {COL4: value6}) - table.put(ROW_KEY2, {COL3: value4}) - table.put(ROW_KEY2, {COL2: value3}) - table.put(ROW_KEY1, {COL2: value2}) - table.put(ROW_KEY3, {COL3: value5}) - table.put(ROW_KEY1, {COL1: value1}) - - # Retrieve all the timestamps so we can filter with them. - scan_result = list(table.scan(include_timestamp=True)) - self.assertEqual(len(scan_result), 3) - row1, row2, row3 = scan_result - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - self.assertEqual(row3[0], ROW_KEY3) - - # Drop the keys now that we have checked. - _, row1 = row1 - _, row2 = row2 - _, row3 = row3 - - # These are numbered in order of insertion, **not** in - # the order of the values. - ts1 = row3[COL4][1] - ts2 = row2[COL3][1] - ts3 = row2[COL2][1] - ts4 = row1[COL2][1] - ts5 = row3[COL3][1] - ts6 = row1[COL1][1] - - self.assertEqual(row1, {COL1: (value1, ts6), COL2: (value2, ts4)}) - self.assertEqual(row2, {COL2: (value3, ts3), COL3: (value4, ts2)}) - self.assertEqual(row3, {COL3: (value5, ts5), COL4: (value6, ts1)}) - - # All cells before ts1 (exclusive) - scan_result_before_ts1 = list(table.scan(timestamp=ts1, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts1, []) - - # All cells before ts2 (inclusive) - scan_result_before_ts2 = list(table.scan(timestamp=ts2 + 1, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts2, [ - (ROW_KEY2, {COL3: (value4, ts2)}), - (ROW_KEY3, {COL4: (value6, ts1)}), - ]) - - # All cells before ts6 (exclusive) - scan_result_before_ts6 = list(table.scan(timestamp=ts6, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts6, [ - (ROW_KEY1, {COL2: (value2, ts4)}), - (ROW_KEY2, {COL2: (value3, ts3), COL3: (value4, ts2)}), - (ROW_KEY3, {COL3: (value5, ts5), COL4: (value6, ts1)}), - ]) - - -class TestTable_put(BaseTableTest): - - def test_put(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - # Check again, but this time with timestamps. - row1 = table.row(ROW_KEY1, include_timestamp=True) - timestamp1 = row1[COL1][1] - timestamp2 = row1[COL2][1] - self.assertEqual(timestamp1, timestamp2) - - row1_data_with_timestamps = {COL1: (value1, timestamp1), - COL2: (value2, timestamp2)} - self.assertEqual(row1, row1_data_with_timestamps) - - @unittest2.skipIf(USING_HBASE, 'HBase fails to write with a timestamp') - def test_put_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - ts = NOW_MILLIS - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data, timestamp=ts) - - # Check again, but this time with timestamps. - row1 = table.row(ROW_KEY1, include_timestamp=True) - row1_data_with_timestamps = {COL1: (value1, ts), - COL2: (value2, ts)} - self.assertEqual(row1, row1_data_with_timestamps) - - @unittest2.skipIf(not USING_HBASE, ('Cloud Bigtable evictions do not seem ' - 'to occur immediately')) - def test_put_versions_restricted(self): - table = get_table() - families = table.families() - - chosen_fam = COL_FAM2 - self.assertEqual(families[chosen_fam]['max_versions'], 1) - chosen_col = COL3 - self.assertTrue(chosen_col.startswith(chosen_fam + ':')) - - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {chosen_col: value1}) - - all_values_before = table.cells(ROW_KEY1, chosen_col, versions=2) - self.assertEqual(all_values_before, [value1]) - - # Putting another value should evict the first one. - table.put(ROW_KEY1, {chosen_col: value2}) - all_values_after = table.cells(ROW_KEY1, chosen_col, versions=2) - self.assertEqual(all_values_after, [value2]) - - @unittest2.skipIf(not USING_HBASE, ('Cloud Bigtable evictions do not seem ' - 'to occur immediately')) - def test_put_ttl_eviction(self): - table = get_table() - # The Thrift API fails to retrieve the TTL for some reason. - if USING_HBASE: - families = FAMILIES - else: - families = table.families() - - cell_ttl = TTL_FOR_TEST - chosen_fam = COL_FAM2 - self.assertEqual(families[chosen_fam]['time_to_live'], cell_ttl) - chosen_col = COL3 - self.assertTrue(chosen_col.startswith(chosen_fam + ':')) - - value1 = 'value1' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {chosen_col: value1}) - - all_values_before = table.cells(ROW_KEY1, chosen_col) - self.assertEqual(all_values_before, [value1]) - - # Make sure we don't sleep for a problematic length. - self.assertTrue(cell_ttl < 10) - # Wait for time-to-live eviction to occur. - time.sleep(cell_ttl + 0.5) - all_values_after = table.cells(ROW_KEY1, chosen_col) - self.assertEqual(all_values_after, []) - - -class TestTable_delete(BaseTableTest): - - def test_delete(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - def test_delete_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {COL2: value2}) - - def test_delete_with_column_family(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2, COL4: value3} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL_FAM1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {COL4: value3}) - - def test_delete_with_columns_family_overlap(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - # First go-around, use [COL_FAM1, COL1] - table.put(ROW_KEY1, row1_data) - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL_FAM1, COL1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - # Second go-around, use [COL1, COL_FAM1] - table.put(ROW_KEY1, row1_data) - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL1, COL_FAM1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - def test_delete_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - - self.assertTrue(ts1 < ts2) - - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1 - 1) - else: - # NOTE: The Cloud Bigtable "Mutation.DeleteFromRow" mutation does - # not support timestamps. Even attempting to send one - # conditionally(via CheckAndMutateRowRequest) deletes the - # entire row. - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1 - 1) - row1_after_early_delete = table.row(ROW_KEY1, include_timestamp=True) - self.assertEqual(row1_after_early_delete, row1) - - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1) - else: - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1) - row1_after_incl_delete = table.row(ROW_KEY1, include_timestamp=True) - self.assertEqual(row1_after_incl_delete, {COL2: (value2, ts2)}) - - def test_delete_with_columns_and_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - - # Delete with conditions that have no matches. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL2]) - row1_after_delete = table.row(ROW_KEY1, include_timestamp=True) - # NOTE: COL2 is still present since it occurs after ts1 and - # COL1 is still present since it is not in `columns`. - self.assertEqual(row1_after_delete, row1) - - # Delete with conditions that have no matches. - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL_FAM1]) - else: - # NOTE: Cloud Bigtable can't use a timestamp with column families - # since "Mutation.DeleteFromFamily" does not include a - # timestamp range. - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL1, COL2]) - row1_delete_fam = table.row(ROW_KEY1, include_timestamp=True) - # NOTE: COL2 is still present since it occurs after ts1 and - # COL1 is still present since it is not in `columns`. - self.assertEqual(row1_delete_fam, {COL2: (value2, ts2)}) - - -class TestTableCounterMethods(BaseTableTest): - - def test_counter_get(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - # Check that the value is set (does not seem to occur on HBase). - if USING_HBASE: - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - else: - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(0)}) - - def test_counter_inc(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - inc_value = 10 - updated_counter = table.counter_inc(ROW_KEY1, COL1, value=inc_value) - self.assertEqual(updated_counter, inc_value) - - # Check that the value is set (does not seem to occur on HBase). - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(inc_value)}) - - def test_counter_dec(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - dec_value = 10 - updated_counter = table.counter_dec(ROW_KEY1, COL1, value=dec_value) - self.assertEqual(updated_counter, -dec_value) - - # Check that the value is set (does not seem to occur on HBase). - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(-dec_value)}) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index db7da7e..0000000 --- a/tox.ini +++ /dev/null @@ -1,61 +0,0 @@ -[tox] -envlist = - py27,cover,lint,docs - -[testenv] -commands = - nosetests -deps = - nose - unittest2 -setenv = - PYTHONPATH = {toxinidir}/_fake_grpc - -[testenv:cover] -basepython = - python2.7 -commands = - nosetests --with-xunit --with-xcoverage --cover-package=gcloud_bigtable --nocapture --cover-erase --cover-tests --cover-branches --cover-min-percentage=100 -deps = - {[testenv]deps} - coverage - nosexcover - -[pep8] -exclude = gcloud_bigtable/_generated/*,docs/conf.py -verbose = 1 - -[testenv:lint] -basepython = - python2.7 -commands = - {toxinidir}/scripts/pep8_on_repo.sh - python run_pylint.py -deps = - pep8 - pylint - unittest2 -passenv = GCLOUD_* - -[testenv:system-tests] -basepython = - python2.7 -commands = - nosetests -v {toxinidir}/system_tests/run.py - nosetests -v {toxinidir}/system_tests/run_happybase.py -deps = - grpcio==0.13.0 - nose - unittest2 -passenv = GOOGLE_* GCLOUD_* GRPC_TRACE -setenv = - -[testenv:docs] -basepython = - python2.7 -commands = - python -c "import shutil; shutil.rmtree('docs/_build', ignore_errors=True)" - sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html -deps = - Sphinx -passenv = SPHINX_RELEASE From f309a52f328995f68a52272cd1722a02ca71c8fa Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 17 May 2016 09:08:28 -0700 Subject: [PATCH 65/65] Updatings docs config for retired status. --- docs/conf.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 60aaa12..515e0a5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,6 @@ from email import message_from_string import os -from pkg_resources import get_distribution import sys import types @@ -68,8 +67,7 @@ # The short X.Y version. # version = '0.0.1' # The full version, including alpha/beta/rc tags. -distro = get_distribution('gcloud_bigtable') -release = os.getenv('SPHINX_RELEASE', distro.version) +release = os.getenv('SPHINX_RELEASE', '0.0.retired') # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -263,8 +261,7 @@ def add_grpc_mock(grpc_mod, subpackage, module_names): # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -metadata = distro.get_metadata(distro.PKG_INFO) -author = message_from_string(metadata).get('Author') +author = 'Google Cloud Platform' latex_documents = [ (master_doc, 'GoogleCloudBigtable.tex', u'Google Cloud Bigtable Documentation', author, 'manual'),