From 0bdca1f1b54da659ffff76b216e37a81208161f7 Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Thu, 30 Sep 2021 02:44:45 +0530 Subject: [PATCH 01/11] changes for copy backup feature --- google/cloud/spanner_v1/backup.py | 44 +++++++++++-- google/cloud/spanner_v1/instance.py | 35 +++++++++++ tests/system/test_backup_api.py | 97 +++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 4 deletions(-) mode change 100644 => 100755 google/cloud/spanner_v1/backup.py mode change 100644 => 100755 google/cloud/spanner_v1/instance.py mode change 100644 => 100755 tests/system/test_backup_api.py diff --git a/google/cloud/spanner_v1/backup.py b/google/cloud/spanner_v1/backup.py old mode 100644 new mode 100755 index dba7ba1fcb..9cf60ece83 --- a/google/cloud/spanner_v1/backup.py +++ b/google/cloud/spanner_v1/backup.py @@ -21,6 +21,8 @@ from google.cloud.spanner_admin_database_v1 import Backup as BackupPB from google.cloud.spanner_admin_database_v1 import CreateBackupEncryptionConfig from google.cloud.spanner_admin_database_v1 import CreateBackupRequest +from google.cloud.spanner_admin_database_v1 import CopyBackupEncryptionConfig +from google.cloud.spanner_admin_database_v1 import CopyBackupRequest from google.cloud.spanner_v1._helpers import _metadata_with_prefix _BACKUP_NAME_RE = re.compile( @@ -74,6 +76,7 @@ def __init__( backup_id, instance, database="", + source_backup=None, expire_time=None, version_time=None, encryption_config=None, @@ -81,6 +84,7 @@ def __init__( self.backup_id = backup_id self._instance = instance self._database = database + self._source_backup = source_backup self._expire_time = expire_time self._create_time = None self._version_time = version_time @@ -89,7 +93,14 @@ def __init__( self._referencing_databases = None self._encryption_info = None if type(encryption_config) == dict: - self._encryption_config = CreateBackupEncryptionConfig(**encryption_config) + if source_backup: + self._encryption_config = CopyBackupEncryptionConfig( + **encryption_config + ) + else: + self._encryption_config = CreateBackupEncryptionConfig( + **encryption_config + ) else: self._encryption_config = encryption_config @@ -223,7 +234,7 @@ def from_pb(cls, backup_pb, instance): return cls(backup_id, instance) def create(self): - """Create this backup within its instance. + """Create this backup or backup copy within its instance. :rtype: :class:`~google.api_core.operation.Operation` :returns: a future used to poll the status of the create request @@ -234,6 +245,32 @@ def create(self): """ if not self._expire_time: raise ValueError("expire_time not set") + + api = self._instance._client.database_admin_api + metadata = _metadata_with_prefix(self.name) + + if self._source_backup: + if ( + self._encryption_config + and self._encryption_config.kms_key_name + and self._encryption_config.encryption_type + != CopyBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION + ): + raise ValueError( + "kms_key_name only used with CUSTOMER_MANAGED_ENCRYPTION" + ) + + request = CopyBackupRequest( + parent=self._instance.name, + backup_id=self.backup_id, + source_backup=self._source_backup, + expire_time=self._expire_time, + encryption_config=self._encryption_config, + ) + + future = api.copy_backup(request=request, metadata=metadata,) + return future + if not self._database: raise ValueError("database not set") if ( @@ -243,8 +280,7 @@ def create(self): != CreateBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION ): raise ValueError("kms_key_name only used with CUSTOMER_MANAGED_ENCRYPTION") - api = self._instance._client.database_admin_api - metadata = _metadata_with_prefix(self.name) + backup = BackupPB( database=self._database, expire_time=self.expire_time, diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py old mode 100644 new mode 100755 index 75e70eaf17..13e3ff82b7 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -551,6 +551,41 @@ def backup( encryption_config=encryption_config, ) + def copy_backup( + self, backup_id, source_backup, expire_time=None, encryption_config=None, + ): + """Factory to create a copy backup within this instance. + + :type backup_id: str + :param backup_id: The ID of the backup copy. + + :type source_backup_id: str + :param backup_id: The ID of the source backup to be copied. + + :type expire_time: :class:`datetime.datetime` + :param expire_time: + Optional. The expire time that will be used when creating the backup. + Required if the create method needs to be called. + + :type encryption_config: + :class:`~google.cloud.spanner_admin_database_v1.types.CreateBackupEncryptionConfig` + or :class:`dict` + :param encryption_config: + (Optional) Encryption configuration for the backup. + If a dict is provided, it must be of the same form as the protobuf + message :class:`~google.cloud.spanner_admin_database_v1.types.CreateBackupEncryptionConfig` + + :rtype: :class:`~google.cloud.spanner_v1.backup.Backup` + :returns: a copy backup owned by this instance. + """ + return Backup( + backup_id, + self, + source_backup=source_backup, + expire_time=expire_time, + encryption_config=encryption_config, + ) + def list_backups(self, filter_="", page_size=None): """List backups for the instance. diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py old mode 100644 new mode 100755 index 59237113e6..f1c7d492d7 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -199,6 +199,80 @@ def test_backup_workflow( assert not backup.exists() +def test_copy_backup_workflow( + shared_instance, shared_database, database_version_time, backups_to_delete, +): + from google.cloud.spanner_admin_database_v1 import ( + CreateBackupEncryptionConfig, + CopyBackupEncryptionConfig, + EncryptionInfo, + ) + + backup_id = _helpers.unique_id("backup_id", separator="_") + source_backup_id = _helpers.unique_id("source_backup_id", separator="_") + expire_time = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + days=3 + ) + copy_encryption_enum = CopyBackupEncryptionConfig.EncryptionType + copy_encryption_config = CopyBackupEncryptionConfig( + encryption_type=copy_encryption_enum.GOOGLE_DEFAULT_ENCRYPTION, + ) + + source_encryption_enum = CreateBackupEncryptionConfig.EncryptionType + source_encryption_config = CreateBackupEncryptionConfig( + encryption_type=source_encryption_enum.GOOGLE_DEFAULT_ENCRYPTION, + ) + + # Create backup. + source_backup = shared_instance.backup( + source_backup_id, + database=shared_database, + expire_time=expire_time, + version_time=database_version_time, + encryption_config=source_encryption_config, + ) + operation = source_backup.create() + backups_to_delete.append(source_backup) + operation.result() # blocks indefinitely + + # Create a copy backup + copy_backup = shared_instance.copy_backup( + backup_id=backup_id, + source_backup=source_backup.name, + expire_time=expire_time, + encryption_config=copy_encryption_config, + ) + operation = copy_backup.create() + backups_to_delete.append(copy_backup) + + # Check metadata. + metadata = operation.metadata + assert copy_backup.name == metadata.name + operation.result() # blocks indefinitely + + # Check backup object. + copy_backup.reload() + assert expire_time == copy_backup.expire_time + assert copy_backup.create_time is not None + assert copy_backup.size_bytes is not None + assert copy_backup.state is not None + assert ( + EncryptionInfo.Type.GOOGLE_DEFAULT_ENCRYPTION + == copy_backup.encryption_info.encryption_type + ) + + # Update with valid argument. + valid_expire_time = datetime.datetime.now( + datetime.timezone.utc + ) + datetime.timedelta(days=7) + copy_backup.update_expire_time(valid_expire_time) + assert valid_expire_time == copy_backup.expire_time + + source_backup.delete() + copy_backup.delete() + assert not copy_backup.exists() + + def test_backup_create_w_version_time_dflt_to_create_time( shared_instance, shared_database, backups_to_delete, databases_to_delete, ): @@ -289,6 +363,29 @@ def test_backup_create_w_invalid_version_time_future( op.result() # blocks indefinitely +def test_copy_backup_create_w_invalid_expire_time(shared_instance, shared_database): + backup_id = _helpers.unique_id("backup_id", separator="_") + source_backup_id = _helpers.unique_id("source_backup_id", separator="_") + valid_expire_time = datetime.datetime.now( + datetime.timezone.utc + ) + datetime.timedelta(days=7) + invalid_expire_time = datetime.datetime.now(datetime.timezone.utc) + + source_backup = shared_instance.backup( + source_backup_id, database=shared_database, expire_time=valid_expire_time + ) + op = source_backup.create() + op.result() # blocks indefinitely + + copy_backup = shared_instance.copy_backup( + backup_id, source_backup_id, expire_time=invalid_expire_time + ) + + with pytest.raises(exceptions.InvalidArgument): + op = copy_backup.create() + op.result() # blocks indefinitely + + def test_database_restore_to_diff_instance( shared_instance, shared_database, From d69a889818bf6aa3e49a4e58a760feb0207e748c Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Thu, 30 Sep 2021 04:00:55 +0530 Subject: [PATCH 02/11] changes to test case --- tests/system/test_backup_api.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py index f1c7d492d7..3a9cb97edf 100755 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -246,8 +246,8 @@ def test_copy_backup_workflow( backups_to_delete.append(copy_backup) # Check metadata. - metadata = operation.metadata - assert copy_backup.name == metadata.name + # metadata = operation.metadata + # assert copy_backup.name == metadata.name operation.result() # blocks indefinitely # Check backup object. @@ -363,7 +363,9 @@ def test_backup_create_w_invalid_version_time_future( op.result() # blocks indefinitely -def test_copy_backup_create_w_invalid_expire_time(shared_instance, shared_database): +def test_copy_backup_create_w_invalid_expire_time( + shared_instance, shared_database, backups_to_delete, +): backup_id = _helpers.unique_id("backup_id", separator="_") source_backup_id = _helpers.unique_id("source_backup_id", separator="_") valid_expire_time = datetime.datetime.now( @@ -376,14 +378,19 @@ def test_copy_backup_create_w_invalid_expire_time(shared_instance, shared_databa ) op = source_backup.create() op.result() # blocks indefinitely + backups_to_delete.append(source_backup) copy_backup = shared_instance.copy_backup( - backup_id, source_backup_id, expire_time=invalid_expire_time + backup_id=backup_id, + source_backup=source_backup.name, + expire_time=invalid_expire_time, ) with pytest.raises(exceptions.InvalidArgument): - op = copy_backup.create() - op.result() # blocks indefinitely + operation = copy_backup.create() + operation.result() # blocks indefinitely + + source_backup.delete() def test_database_restore_to_diff_instance( From 22fb08ece2179fd9750a03ee4ed26d05531fa245 Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Thu, 30 Sep 2021 04:05:57 +0530 Subject: [PATCH 03/11] changes to documenttation --- google/cloud/spanner_v1/instance.py | 6 +++--- tests/system/test_backup_api.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index 13e3ff82b7..f6787efcb9 100755 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -564,16 +564,16 @@ def copy_backup( :type expire_time: :class:`datetime.datetime` :param expire_time: - Optional. The expire time that will be used when creating the backup. + Optional. The expire time that will be used when creating the copy backup. Required if the create method needs to be called. :type encryption_config: - :class:`~google.cloud.spanner_admin_database_v1.types.CreateBackupEncryptionConfig` + :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` or :class:`dict` :param encryption_config: (Optional) Encryption configuration for the backup. If a dict is provided, it must be of the same form as the protobuf - message :class:`~google.cloud.spanner_admin_database_v1.types.CreateBackupEncryptionConfig` + message :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` :rtype: :class:`~google.cloud.spanner_v1.backup.Backup` :returns: a copy backup owned by this instance. diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py index 3a9cb97edf..dc2e1cbebc 100755 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -246,8 +246,8 @@ def test_copy_backup_workflow( backups_to_delete.append(copy_backup) # Check metadata. - # metadata = operation.metadata - # assert copy_backup.name == metadata.name + metadata = operation.metadata + assert copy_backup.name == metadata.name operation.result() # blocks indefinitely # Check backup object. From cf138b7aa9e508afd7ed40d527029848f22fc619 Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Fri, 24 Dec 2021 19:01:15 +0530 Subject: [PATCH 04/11] feat: changes as per review, adding shared_backup --- google/cloud/spanner_v1/backup.py | 26 +++++++++++++++++++- tests/system/_helpers.py | 3 +++ tests/system/conftest.py | 33 +++++++++++++++++++++++++ tests/system/test_backup_api.py | 40 +++++-------------------------- 4 files changed, 67 insertions(+), 35 deletions(-) mode change 100644 => 100755 tests/system/_helpers.py mode change 100644 => 100755 tests/system/conftest.py diff --git a/google/cloud/spanner_v1/backup.py b/google/cloud/spanner_v1/backup.py index 9cf60ece83..81e13da4cc 100755 --- a/google/cloud/spanner_v1/backup.py +++ b/google/cloud/spanner_v1/backup.py @@ -76,10 +76,10 @@ def __init__( backup_id, instance, database="", - source_backup=None, expire_time=None, version_time=None, encryption_config=None, + source_backup=None, ): self.backup_id = backup_id self._instance = instance @@ -92,6 +92,8 @@ def __init__( self._state = None self._referencing_databases = None self._encryption_info = None + self._max_expire_time = None + self._referencing_backups = None if type(encryption_config) == dict: if source_backup: self._encryption_config = CopyBackupEncryptionConfig( @@ -196,6 +198,26 @@ def encryption_info(self): """ return self._encryption_info + @property + def max_expire_time(self): + """The max allowed expiration time of the backup. + + :rtype: :class:`datetime.datetime` + :returns: a datetime object representing the max expire time of + this backup + """ + return self._max_expire_time + + @property + def referencing_backups(self): + """The names of the destination backups being created by copying this source backup. + + :rtype: list of strings + :returns: a list of backup path strings which specify the backups that are + referencing this copy backup + """ + return self._referencing_backups + @classmethod def from_pb(cls, backup_pb, instance): """Create an instance of this class from a protobuf message. @@ -330,6 +352,8 @@ def reload(self): self._state = BackupPB.State(pb.state) self._referencing_databases = pb.referencing_databases self._encryption_info = pb.encryption_info + self._max_expire_time = pb.max_expire_time + self._referencing_backups = pb.referencing_backups def update_expire_time(self, new_expire_time): """Update the expire time of this backup. diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py old mode 100644 new mode 100755 index 0baff62433..40fbdb5364 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -39,6 +39,9 @@ DATABASE_OPERATION_TIMEOUT_IN_SECONDS = int( os.getenv("SPANNER_DATABASE_OPERATION_TIMEOUT_IN_SECONDS", 60) ) +BACKUP_OPERATION_TIMEOUT_IN_SECONDS = int( + os.getenv("SPANNER_BACKUP_OPERATION_TIMEOUT_IN_SECONDS", 1200) +) USE_EMULATOR_ENVVAR = "SPANNER_EMULATOR_HOST" USE_EMULATOR = os.getenv(USE_EMULATOR_ENVVAR) is not None diff --git a/tests/system/conftest.py b/tests/system/conftest.py old mode 100644 new mode 100755 index 3a8c973f1b..93a27a5665 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import time +from google.cloud.spanner_admin_database_v1.types.backup import ( + CreateBackupEncryptionConfig, +) import pytest @@ -67,6 +71,11 @@ def database_operation_timeout(): return _helpers.DATABASE_OPERATION_TIMEOUT_IN_SECONDS +@pytest.fixture(scope="session") +def backup_operation_timeout(): + return _helpers.BACKUP_OPERATION_TIMEOUT_IN_SECONDS + + @pytest.fixture(scope="session") def shared_instance_id(): if _helpers.CREATE_INSTANCE: @@ -148,6 +157,30 @@ def shared_database(shared_instance, database_operation_timeout): database.drop() +@pytest.fixture(scope="session") +def shared_backup(shared_instance, shared_database, backup_operation_timeout): + backup_name = _helpers.unique_id("test_backup") + expire_time = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + days=3 + ) + source_encryption_enum = CreateBackupEncryptionConfig.EncryptionType + source_encryption_config = CreateBackupEncryptionConfig( + encryption_type=source_encryption_enum.GOOGLE_DEFAULT_ENCRYPTION, + ) + backup = shared_instance.backup( + backup_name, + database=shared_database, + expire_time=expire_time, + encryption_config=source_encryption_config, + ) + operation = backup.create() + operation.result(backup_operation_timeout) # raises on failure / timeout. + + yield backup + + backup.delete() + + @pytest.fixture(scope="function") def databases_to_delete(): to_delete = [] diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py index dc2e1cbebc..48614fe72f 100755 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -200,7 +200,7 @@ def test_backup_workflow( def test_copy_backup_workflow( - shared_instance, shared_database, database_version_time, backups_to_delete, + shared_instance, shared_backup, backups_to_delete, ): from google.cloud.spanner_admin_database_v1 import ( CreateBackupEncryptionConfig, @@ -209,7 +209,6 @@ def test_copy_backup_workflow( ) backup_id = _helpers.unique_id("backup_id", separator="_") - source_backup_id = _helpers.unique_id("source_backup_id", separator="_") expire_time = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( days=3 ) @@ -218,27 +217,12 @@ def test_copy_backup_workflow( encryption_type=copy_encryption_enum.GOOGLE_DEFAULT_ENCRYPTION, ) - source_encryption_enum = CreateBackupEncryptionConfig.EncryptionType - source_encryption_config = CreateBackupEncryptionConfig( - encryption_type=source_encryption_enum.GOOGLE_DEFAULT_ENCRYPTION, - ) - # Create backup. - source_backup = shared_instance.backup( - source_backup_id, - database=shared_database, - expire_time=expire_time, - version_time=database_version_time, - encryption_config=source_encryption_config, - ) - operation = source_backup.create() - backups_to_delete.append(source_backup) - operation.result() # blocks indefinitely - + shared_backup.reload() # Create a copy backup copy_backup = shared_instance.copy_backup( backup_id=backup_id, - source_backup=source_backup.name, + source_backup=shared_backup.name, expire_time=expire_time, encryption_config=copy_encryption_config, ) @@ -268,7 +252,6 @@ def test_copy_backup_workflow( copy_backup.update_expire_time(valid_expire_time) assert valid_expire_time == copy_backup.expire_time - source_backup.delete() copy_backup.delete() assert not copy_backup.exists() @@ -364,25 +347,16 @@ def test_backup_create_w_invalid_version_time_future( def test_copy_backup_create_w_invalid_expire_time( - shared_instance, shared_database, backups_to_delete, + shared_instance, shared_backup, ): backup_id = _helpers.unique_id("backup_id", separator="_") - source_backup_id = _helpers.unique_id("source_backup_id", separator="_") - valid_expire_time = datetime.datetime.now( - datetime.timezone.utc - ) + datetime.timedelta(days=7) invalid_expire_time = datetime.datetime.now(datetime.timezone.utc) - source_backup = shared_instance.backup( - source_backup_id, database=shared_database, expire_time=valid_expire_time - ) - op = source_backup.create() - op.result() # blocks indefinitely - backups_to_delete.append(source_backup) + shared_backup.reload() copy_backup = shared_instance.copy_backup( backup_id=backup_id, - source_backup=source_backup.name, + source_backup=shared_backup.name, expire_time=invalid_expire_time, ) @@ -390,8 +364,6 @@ def test_copy_backup_create_w_invalid_expire_time( operation = copy_backup.create() operation.result() # blocks indefinitely - source_backup.delete() - def test_database_restore_to_diff_instance( shared_instance, From 9d6f8ecd515f4ee704a1855dcb18a5503024629e Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Wed, 2 Feb 2022 20:26:38 +0530 Subject: [PATCH 05/11] changes for cross region backup --- google/cloud/spanner_v1/backup.py | 2 -- google/cloud/spanner_v1/instance.py | 9 ++------ samples/samples/backup_sample.py | 33 ++++++++++++++++++++++++++- samples/samples/backup_sample_test.py | 18 ++++++++++++++- tests/system/conftest.py | 3 +++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/google/cloud/spanner_v1/backup.py b/google/cloud/spanner_v1/backup.py index 81e13da4cc..1abf9d1df4 100755 --- a/google/cloud/spanner_v1/backup.py +++ b/google/cloud/spanner_v1/backup.py @@ -201,7 +201,6 @@ def encryption_info(self): @property def max_expire_time(self): """The max allowed expiration time of the backup. - :rtype: :class:`datetime.datetime` :returns: a datetime object representing the max expire time of this backup @@ -211,7 +210,6 @@ def max_expire_time(self): @property def referencing_backups(self): """The names of the destination backups being created by copying this source backup. - :rtype: list of strings :returns: a list of backup path strings which specify the backups that are referencing this copy backup diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index f6787efcb9..454c8c72b2 100755 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -555,18 +555,14 @@ def copy_backup( self, backup_id, source_backup, expire_time=None, encryption_config=None, ): """Factory to create a copy backup within this instance. - :type backup_id: str :param backup_id: The ID of the backup copy. - - :type source_backup_id: str - :param backup_id: The ID of the source backup to be copied. - + :type source_backup: str + :param source_backup_id: The full path of the source backup to be copied. :type expire_time: :class:`datetime.datetime` :param expire_time: Optional. The expire time that will be used when creating the copy backup. Required if the create method needs to be called. - :type encryption_config: :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` or :class:`dict` @@ -574,7 +570,6 @@ def copy_backup( (Optional) Encryption configuration for the backup. If a dict is provided, it must be of the same form as the protobuf message :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` - :rtype: :class:`~google.cloud.spanner_v1.backup.Backup` :returns: a copy backup owned by this instance. """ diff --git a/samples/samples/backup_sample.py b/samples/samples/backup_sample.py index d22530c735..eca715438d 100644 --- a/samples/samples/backup_sample.py +++ b/samples/samples/backup_sample.py @@ -330,7 +330,8 @@ def update_backup(instance_id, backup_id): # Expire time must be within 366 days of the create time of the backup. old_expire_time = backup.expire_time - new_expire_time = old_expire_time + timedelta(days=30) + # New expire time should be less than the max expire time + new_expire_time = min(backup.max_expire_time, old_expire_time + timedelta(days=30)) backup.update_expire_time(new_expire_time) print( "Backup {} expire time was updated from {} to {}.".format( @@ -381,6 +382,33 @@ def create_database_with_version_retention_period(instance_id, database_id, rete # [END spanner_create_database_with_version_retention_period] +# [START spanner_copy_backup] +def copy_backup(instance_id, backup_id, source_backup_path): + """Copies a backup.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # Create a backup object and wait for copy backup operation to complete. + expire_time = datetime.utcnow() + timedelta(days=14) + copy_backup = instance.copy_backup(backup_id, source_backup=source_backup_path, expire_time=expire_time) + operation = copy_backup.create() + + # Wait for copy backup operation to complete. + operation.result(2100) + + # Verify that the copy backup is ready. + copy_backup.reload() + assert copy_backup.is_ready() is True + + print( + "Backup {} of size {} bytes was created at {} with version time {}".format( + copy_backup.name, copy_backup.size_bytes, copy_backup.create_time, copy_backup.version_time, + ) + ) + +# [END spanner_copy_backup] + + if __name__ == "__main__": # noqa: C901 parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -404,6 +432,7 @@ def create_database_with_version_retention_period(instance_id, database_id, rete "list_database_operations", help=list_database_operations.__doc__ ) subparsers.add_parser("delete_backup", help=delete_backup.__doc__) + subparsers.add_parser("copy_backup", help=copy_backup.__doc__) args = parser.parse_args() @@ -423,5 +452,7 @@ def create_database_with_version_retention_period(instance_id, database_id, rete list_database_operations(args.instance_id) elif args.command == "delete_backup": delete_backup(args.instance_id, args.backup_id) + elif args.command == "copy_backup": + copy_backup(args.instance_id, args.backup_id, args.source_backup_id) else: print("Command {} did not match expected commands.".format(args.command)) diff --git a/samples/samples/backup_sample_test.py b/samples/samples/backup_sample_test.py index 6d89dcf440..d3ecabbe9c 100644 --- a/samples/samples/backup_sample_test.py +++ b/samples/samples/backup_sample_test.py @@ -41,6 +41,7 @@ def unique_backup_id(): CMEK_BACKUP_ID = unique_backup_id() RETENTION_DATABASE_ID = unique_database_id() RETENTION_PERIOD = "7d" +COPY_BACKUP_ID = unique_backup_id() @pytest.mark.dependency(name="create_backup") @@ -125,11 +126,14 @@ def test_update_backup(capsys, instance_id): assert BACKUP_ID in out -@pytest.mark.dependency(depends=["create_backup"]) +@pytest.mark.dependency(depends=["create_backup","copy_backup"]) def test_delete_backup(capsys, instance_id): backup_sample.delete_backup(instance_id, BACKUP_ID) out, _ = capsys.readouterr() assert BACKUP_ID in out + backup_sample.delete_backup(instance_id, COPY_BACKUP_ID) + out, _ = capsys.readouterr() + assert COPY_BACKUP_ID in out @pytest.mark.dependency(depends=["create_backup"]) @@ -155,3 +159,15 @@ def test_create_database_with_retention_period(capsys, sample_instance): assert ("retention period " + RETENTION_PERIOD) in out database = sample_instance.database(RETENTION_DATABASE_ID) database.drop() + +@pytest.mark.dependency(name="copy_backup",depends=["create_backup"]) +def test_copy_backup(capsys, instance_id, spanner_client): + source_backp_path=spanner_client.project_name+'/instances/'+instance_id+'/backups/'+BACKUP_ID + backup_sample.copy_backup( + instance_id, + source_backp_path, + BACKUP_ID + ) + out, _ = capsys.readouterr() + assert COPY_BACKUP_ID in out + diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 599b6961c4..868c209151 100755 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -22,6 +22,9 @@ from google.cloud import spanner_v1 from . import _helpers +from google.cloud.spanner_admin_database_v1.types.backup import ( + CreateBackupEncryptionConfig, +) @pytest.fixture(scope="function") From 7078b98cbbd85dcb177c8fcffb3ebd5869c7e2a0 Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Tue, 15 Mar 2022 13:45:45 +0530 Subject: [PATCH 06/11] samples: changes to list backup operations --- samples/samples/backup_sample.py | 19 +++++++++++++++++-- samples/samples/backup_sample_test.py | 8 +++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/samples/samples/backup_sample.py b/samples/samples/backup_sample.py index eca715438d..66ab8d3e19 100644 --- a/samples/samples/backup_sample.py +++ b/samples/samples/backup_sample.py @@ -192,7 +192,7 @@ def cancel_backup(instance_id, database_id, backup_id): # [START spanner_list_backup_operations] -def list_backup_operations(instance_id, database_id): +def list_backup_operations(instance_id, database_id, backup_id): spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) @@ -210,6 +210,21 @@ def list_backup_operations(instance_id, database_id): metadata.name, metadata.database, metadata.progress.progress_percent ) ) + + # List the CopyBackup operations. + filter_ = ( + "(metadata.@type:type.googleapis.com/" + "google.spanner.admin.database.v1.CopyBackupMetadata)" + " AND (metadata.source_backup:{})" + ).format(backup_id) + operations = instance.list_backup_operations(filter_=filter_) + for op in operations: + metadata = op.metadata + print( + "Backup {} on source backup {}: {}% complete.".format( + metadata.name, metadata.source_backup, metadata.progress.progress_percent + ) + ) # [END spanner_list_backup_operations] @@ -447,7 +462,7 @@ def copy_backup(instance_id, backup_id, source_backup_path): elif args.command == "list_backups": list_backups(args.instance_id, args.database_id, args.backup_id) elif args.command == "list_backup_operations": - list_backup_operations(args.instance_id, args.database_id) + list_backup_operations(args.instance_id, args.database_id, args.backup_id) elif args.command == "list_database_operations": list_database_operations(args.instance_id) elif args.command == "delete_backup": diff --git a/samples/samples/backup_sample_test.py b/samples/samples/backup_sample_test.py index d3ecabbe9c..f53499c362 100644 --- a/samples/samples/backup_sample_test.py +++ b/samples/samples/backup_sample_test.py @@ -100,17 +100,19 @@ def test_restore_database_with_encryption_key( assert kms_key_name in out -@pytest.mark.dependency(depends=["create_backup"]) +@pytest.mark.dependency(depends=["create_backup", "copy_backup"]) def test_list_backup_operations(capsys, instance_id, sample_database): backup_sample.list_backup_operations( - instance_id, sample_database.database_id) + instance_id, sample_database.database_id, BACKUP_ID) out, _ = capsys.readouterr() assert BACKUP_ID in out assert sample_database.database_id in out + assert COPY_BACKUP_ID in out + print(out) @pytest.mark.dependency(depends=["create_backup"]) -def test_list_backups(capsys, instance_id, sample_database): +def test_list_backups(capsys, instance_id, sample_database, ): backup_sample.list_backups( instance_id, sample_database.database_id, BACKUP_ID, ) From 0b0dc69600bea719d3fae5878c792e44bbaa7431 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sun, 13 Mar 2022 20:53:27 +0100 Subject: [PATCH 07/11] chore(deps): update all dependencies (#689) --- .github/workflows/integration-tests-against-emulator.yaml | 4 ++-- samples/samples/requirements-test.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index 7438f8f0a9..3c8b1c5080 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -17,9 +17,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.8 - name: Install nox diff --git a/samples/samples/requirements-test.txt b/samples/samples/requirements-test.txt index b8e7474e10..47ad2792b2 100644 --- a/samples/samples/requirements-test.txt +++ b/samples/samples/requirements-test.txt @@ -1,4 +1,4 @@ -pytest==7.0.1 +pytest==7.1.0 pytest-dependency==0.5.1 mock==4.0.3 google-cloud-testutils==1.3.1 From 3e7b2c5bb6f477416165b6c5d64ff19e1052a9c0 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Sat, 19 Mar 2022 11:34:15 +0100 Subject: [PATCH 08/11] chore(deps): update dependency pytest to v7.1.1 (#690) --- samples/samples/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/samples/requirements-test.txt b/samples/samples/requirements-test.txt index 47ad2792b2..3d42f3a24a 100644 --- a/samples/samples/requirements-test.txt +++ b/samples/samples/requirements-test.txt @@ -1,4 +1,4 @@ -pytest==7.1.0 +pytest==7.1.1 pytest-dependency==0.5.1 mock==4.0.3 google-cloud-testutils==1.3.1 From 42f3eebd1a6d400d215103bf0fd0313fb785c70d Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 17:03:18 -0400 Subject: [PATCH 09/11] feat: add support for Cross region backup proto changes (#691) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Synchronize new proto/yaml changes. PiperOrigin-RevId: 436114471 Source-Link: https://github.com/googleapis/googleapis/commit/6379d5fe706781af6682447f77f20d18b4db05b2 Source-Link: https://github.com/googleapis/googleapis-gen/commit/a59984b4cb711eeb186bca4f5b35adbfe60825df Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYTU5OTg0YjRjYjcxMWVlYjE4NmJjYTRmNWIzNWFkYmZlNjA4MjVkZiJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .../spanner_admin_database_v1/__init__.py | 6 + .../gapic_metadata.json | 10 + .../services/database_admin/async_client.py | 162 ++++++++++++ .../services/database_admin/client.py | 162 ++++++++++++ .../database_admin/transports/base.py | 12 + .../database_admin/transports/grpc.py | 38 +++ .../database_admin/transports/grpc_asyncio.py | 38 +++ .../types/__init__.py | 6 + .../spanner_admin_database_v1/types/backup.py | 185 ++++++++++++- .../types/spanner_database_admin.py | 2 + ...et_metadata_spanner admin database_v1.json | 89 +++++++ ...erated_database_admin_copy_backup_async.py | 51 ++++ ...nerated_database_admin_copy_backup_sync.py | 51 ++++ ...ixup_spanner_admin_database_v1_keywords.py | 1 + .../test_database_admin.py | 244 ++++++++++++++++++ 15 files changed, 1050 insertions(+), 7 deletions(-) create mode 100644 samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_async.py create mode 100644 samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_sync.py diff --git a/google/cloud/spanner_admin_database_v1/__init__.py b/google/cloud/spanner_admin_database_v1/__init__.py index e587590c9a..ee52bda123 100644 --- a/google/cloud/spanner_admin_database_v1/__init__.py +++ b/google/cloud/spanner_admin_database_v1/__init__.py @@ -19,6 +19,9 @@ from .types.backup import Backup from .types.backup import BackupInfo +from .types.backup import CopyBackupEncryptionConfig +from .types.backup import CopyBackupMetadata +from .types.backup import CopyBackupRequest from .types.backup import CreateBackupEncryptionConfig from .types.backup import CreateBackupMetadata from .types.backup import CreateBackupRequest @@ -57,6 +60,9 @@ "DatabaseAdminAsyncClient", "Backup", "BackupInfo", + "CopyBackupEncryptionConfig", + "CopyBackupMetadata", + "CopyBackupRequest", "CreateBackupEncryptionConfig", "CreateBackupMetadata", "CreateBackupRequest", diff --git a/google/cloud/spanner_admin_database_v1/gapic_metadata.json b/google/cloud/spanner_admin_database_v1/gapic_metadata.json index 1460097dc3..f7272318ef 100644 --- a/google/cloud/spanner_admin_database_v1/gapic_metadata.json +++ b/google/cloud/spanner_admin_database_v1/gapic_metadata.json @@ -10,6 +10,11 @@ "grpc": { "libraryClient": "DatabaseAdminClient", "rpcs": { + "CopyBackup": { + "methods": [ + "copy_backup" + ] + }, "CreateBackup": { "methods": [ "create_backup" @@ -100,6 +105,11 @@ "grpc-async": { "libraryClient": "DatabaseAdminAsyncClient", "rpcs": { + "CopyBackup": { + "methods": [ + "copy_backup" + ] + }, "CreateBackup": { "methods": [ "create_backup" diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/async_client.py b/google/cloud/spanner_admin_database_v1/services/database_admin/async_client.py index add0829bc8..e4793ae26b 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/async_client.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/async_client.py @@ -1506,6 +1506,168 @@ def sample_create_backup(): # Done; return the response. return response + async def copy_backup( + self, + request: Union[backup.CopyBackupRequest, dict] = None, + *, + parent: str = None, + backup_id: str = None, + source_backup: str = None, + expire_time: timestamp_pb2.Timestamp = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation_async.AsyncOperation: + r"""Starts copying a Cloud Spanner Backup. The returned backup + [long-running operation][google.longrunning.Operation] will have + a name of the format + ``projects//instances//backups//operations/`` + and can be used to track copying of the backup. The operation is + associated with the destination backup. The + [metadata][google.longrunning.Operation.metadata] field type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata]. + The [response][google.longrunning.Operation.response] field type + is [Backup][google.spanner.admin.database.v1.Backup], if + successful. Cancelling the returned operation will stop the + copying and delete the backup. Concurrent CopyBackup requests + can run on the same source backup. + + + .. code-block:: python + + from google.cloud import spanner_admin_database_v1 + + def sample_copy_backup(): + # Create a client + client = spanner_admin_database_v1.DatabaseAdminClient() + + # Initialize request argument(s) + request = spanner_admin_database_v1.CopyBackupRequest( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + ) + + # Make the request + operation = client.copy_backup(request=request) + + print("Waiting for operation to complete...") + + response = operation.result() + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.spanner_admin_database_v1.types.CopyBackupRequest, dict]): + The request object. The request for + [CopyBackup][google.spanner.admin.database.v1.DatabaseAdmin.CopyBackup]. + parent (:class:`str`): + Required. The name of the destination instance that will + contain the backup copy. Values are of the form: + ``projects//instances/``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + backup_id (:class:`str`): + Required. The id of the backup copy. The ``backup_id`` + appended to ``parent`` forms the full backup_uri of the + form + ``projects//instances//backups/``. + + This corresponds to the ``backup_id`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + source_backup (:class:`str`): + Required. The source backup to be copied. The source + backup needs to be in READY state for it to be copied. + Once CopyBackup is in progress, the source backup cannot + be deleted or cleaned up on expiration until CopyBackup + is finished. Values are of the form: + ``projects//instances//backups/``. + + This corresponds to the ``source_backup`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + expire_time (:class:`google.protobuf.timestamp_pb2.Timestamp`): + Required. The expiration time of the backup in + microsecond granularity. The expiration time must be at + least 6 hours and at most 366 days from the + ``create_time`` of the source backup. Once the + ``expire_time`` has passed, the backup is eligible to be + automatically deleted by Cloud Spanner to free the + resources used by the backup. + + This corresponds to the ``expire_time`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation_async.AsyncOperation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.spanner_admin_database_v1.types.Backup` + A backup of a Cloud Spanner database. + + """ + # Create or coerce a protobuf request object. + # Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, backup_id, source_backup, expire_time]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = backup.CopyBackupRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if backup_id is not None: + request.backup_id = backup_id + if source_backup is not None: + request.source_backup = source_backup + if expire_time is not None: + request.expire_time = expire_time + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.copy_backup, + default_timeout=3600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation_async.from_gapic( + response, + self._client._transport.operations_client, + backup.Backup, + metadata_type=backup.CopyBackupMetadata, + ) + + # Done; return the response. + return response + async def get_backup( self, request: Union[backup.GetBackupRequest, dict] = None, diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/client.py b/google/cloud/spanner_admin_database_v1/services/database_admin/client.py index 120dec124a..a7106d7aa7 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/client.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/client.py @@ -1704,6 +1704,168 @@ def sample_create_backup(): # Done; return the response. return response + def copy_backup( + self, + request: Union[backup.CopyBackupRequest, dict] = None, + *, + parent: str = None, + backup_id: str = None, + source_backup: str = None, + expire_time: timestamp_pb2.Timestamp = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation.Operation: + r"""Starts copying a Cloud Spanner Backup. The returned backup + [long-running operation][google.longrunning.Operation] will have + a name of the format + ``projects//instances//backups//operations/`` + and can be used to track copying of the backup. The operation is + associated with the destination backup. The + [metadata][google.longrunning.Operation.metadata] field type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata]. + The [response][google.longrunning.Operation.response] field type + is [Backup][google.spanner.admin.database.v1.Backup], if + successful. Cancelling the returned operation will stop the + copying and delete the backup. Concurrent CopyBackup requests + can run on the same source backup. + + + .. code-block:: python + + from google.cloud import spanner_admin_database_v1 + + def sample_copy_backup(): + # Create a client + client = spanner_admin_database_v1.DatabaseAdminClient() + + # Initialize request argument(s) + request = spanner_admin_database_v1.CopyBackupRequest( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + ) + + # Make the request + operation = client.copy_backup(request=request) + + print("Waiting for operation to complete...") + + response = operation.result() + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.spanner_admin_database_v1.types.CopyBackupRequest, dict]): + The request object. The request for + [CopyBackup][google.spanner.admin.database.v1.DatabaseAdmin.CopyBackup]. + parent (str): + Required. The name of the destination instance that will + contain the backup copy. Values are of the form: + ``projects//instances/``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + backup_id (str): + Required. The id of the backup copy. The ``backup_id`` + appended to ``parent`` forms the full backup_uri of the + form + ``projects//instances//backups/``. + + This corresponds to the ``backup_id`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + source_backup (str): + Required. The source backup to be copied. The source + backup needs to be in READY state for it to be copied. + Once CopyBackup is in progress, the source backup cannot + be deleted or cleaned up on expiration until CopyBackup + is finished. Values are of the form: + ``projects//instances//backups/``. + + This corresponds to the ``source_backup`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Required. The expiration time of the backup in + microsecond granularity. The expiration time must be at + least 6 hours and at most 366 days from the + ``create_time`` of the source backup. Once the + ``expire_time`` has passed, the backup is eligible to be + automatically deleted by Cloud Spanner to free the + resources used by the backup. + + This corresponds to the ``expire_time`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation.Operation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.spanner_admin_database_v1.types.Backup` + A backup of a Cloud Spanner database. + + """ + # Create or coerce a protobuf request object. + # Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, backup_id, source_backup, expire_time]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a backup.CopyBackupRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, backup.CopyBackupRequest): + request = backup.CopyBackupRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if backup_id is not None: + request.backup_id = backup_id + if source_backup is not None: + request.source_backup = source_backup + if expire_time is not None: + request.expire_time = expire_time + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.copy_backup] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation.from_gapic( + response, + self._transport.operations_client, + backup.Backup, + metadata_type=backup.CopyBackupMetadata, + ) + + # Done; return the response. + return response + def get_backup( self, request: Union[backup.GetBackupRequest, dict] = None, diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/base.py b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/base.py index 090e2a954e..18dfc4074c 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/base.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/base.py @@ -234,6 +234,9 @@ def _prep_wrapped_messages(self, client_info): self.create_backup: gapic_v1.method.wrap_method( self.create_backup, default_timeout=3600.0, client_info=client_info, ), + self.copy_backup: gapic_v1.method.wrap_method( + self.copy_backup, default_timeout=3600.0, client_info=client_info, + ), self.get_backup: gapic_v1.method.wrap_method( self.get_backup, default_retry=retries.Retry( @@ -444,6 +447,15 @@ def create_backup( ]: raise NotImplementedError() + @property + def copy_backup( + self, + ) -> Callable[ + [backup.CopyBackupRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + @property def get_backup( self, diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py index 9c0d1ea4d0..6f1d695122 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc.py @@ -593,6 +593,44 @@ def create_backup( ) return self._stubs["create_backup"] + @property + def copy_backup( + self, + ) -> Callable[[backup.CopyBackupRequest], operations_pb2.Operation]: + r"""Return a callable for the copy backup method over gRPC. + + Starts copying a Cloud Spanner Backup. The returned backup + [long-running operation][google.longrunning.Operation] will have + a name of the format + ``projects//instances//backups//operations/`` + and can be used to track copying of the backup. The operation is + associated with the destination backup. The + [metadata][google.longrunning.Operation.metadata] field type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata]. + The [response][google.longrunning.Operation.response] field type + is [Backup][google.spanner.admin.database.v1.Backup], if + successful. Cancelling the returned operation will stop the + copying and delete the backup. Concurrent CopyBackup requests + can run on the same source backup. + + Returns: + Callable[[~.CopyBackupRequest], + ~.Operation]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "copy_backup" not in self._stubs: + self._stubs["copy_backup"] = self.grpc_channel.unary_unary( + "/google.spanner.admin.database.v1.DatabaseAdmin/CopyBackup", + request_serializer=backup.CopyBackupRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["copy_backup"] + @property def get_backup(self) -> Callable[[backup.GetBackupRequest], backup.Backup]: r"""Return a callable for the get backup method over gRPC. diff --git a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py index fd35a3eaf5..2a3200a882 100644 --- a/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py +++ b/google/cloud/spanner_admin_database_v1/services/database_admin/transports/grpc_asyncio.py @@ -604,6 +604,44 @@ def create_backup( ) return self._stubs["create_backup"] + @property + def copy_backup( + self, + ) -> Callable[[backup.CopyBackupRequest], Awaitable[operations_pb2.Operation]]: + r"""Return a callable for the copy backup method over gRPC. + + Starts copying a Cloud Spanner Backup. The returned backup + [long-running operation][google.longrunning.Operation] will have + a name of the format + ``projects//instances//backups//operations/`` + and can be used to track copying of the backup. The operation is + associated with the destination backup. The + [metadata][google.longrunning.Operation.metadata] field type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata]. + The [response][google.longrunning.Operation.response] field type + is [Backup][google.spanner.admin.database.v1.Backup], if + successful. Cancelling the returned operation will stop the + copying and delete the backup. Concurrent CopyBackup requests + can run on the same source backup. + + Returns: + Callable[[~.CopyBackupRequest], + Awaitable[~.Operation]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "copy_backup" not in self._stubs: + self._stubs["copy_backup"] = self.grpc_channel.unary_unary( + "/google.spanner.admin.database.v1.DatabaseAdmin/CopyBackup", + request_serializer=backup.CopyBackupRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["copy_backup"] + @property def get_backup( self, diff --git a/google/cloud/spanner_admin_database_v1/types/__init__.py b/google/cloud/spanner_admin_database_v1/types/__init__.py index 8a7e38d1ab..8d4b5f4094 100644 --- a/google/cloud/spanner_admin_database_v1/types/__init__.py +++ b/google/cloud/spanner_admin_database_v1/types/__init__.py @@ -16,6 +16,9 @@ from .backup import ( Backup, BackupInfo, + CopyBackupEncryptionConfig, + CopyBackupMetadata, + CopyBackupRequest, CreateBackupEncryptionConfig, CreateBackupMetadata, CreateBackupRequest, @@ -58,6 +61,9 @@ __all__ = ( "Backup", "BackupInfo", + "CopyBackupEncryptionConfig", + "CopyBackupMetadata", + "CopyBackupRequest", "CreateBackupEncryptionConfig", "CreateBackupMetadata", "CreateBackupRequest", diff --git a/google/cloud/spanner_admin_database_v1/types/backup.py b/google/cloud/spanner_admin_database_v1/types/backup.py index da5f4d4b2e..b4cff201a2 100644 --- a/google/cloud/spanner_admin_database_v1/types/backup.py +++ b/google/cloud/spanner_admin_database_v1/types/backup.py @@ -27,6 +27,8 @@ "Backup", "CreateBackupRequest", "CreateBackupMetadata", + "CopyBackupRequest", + "CopyBackupMetadata", "UpdateBackupRequest", "GetBackupRequest", "DeleteBackupRequest", @@ -36,6 +38,7 @@ "ListBackupOperationsResponse", "BackupInfo", "CreateBackupEncryptionConfig", + "CopyBackupEncryptionConfig", }, ) @@ -107,6 +110,23 @@ class Backup(proto.Message): database_dialect (google.cloud.spanner_admin_database_v1.types.DatabaseDialect): Output only. The database dialect information for the backup. + referencing_backups (Sequence[str]): + Output only. The names of the destination backups being + created by copying this source backup. The backup names are + of the form + ``projects//instances//backups/``. + Referencing backups may exist in different instances. The + existence of any referencing backup prevents the backup from + being deleted. When the copy operation is done (either + successfully completed or cancelled or the destination + backup is deleted), the reference to the backup is removed. + max_expire_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. The max allowed expiration time of the backup, + with microseconds granularity. A backup's expiration time + can be configured in multiple APIs: CreateBackup, + UpdateBackup, CopyBackup. When updating or copying an + existing backup, the expiration time specified must be less + than ``Backup.max_expire_time``. """ class State(proto.Enum): @@ -129,6 +149,10 @@ class State(proto.Enum): proto.MESSAGE, number=8, message=common.EncryptionInfo, ) database_dialect = proto.Field(proto.ENUM, number=10, enum=common.DatabaseDialect,) + referencing_backups = proto.RepeatedField(proto.STRING, number=11,) + max_expire_time = proto.Field( + proto.MESSAGE, number=12, message=timestamp_pb2.Timestamp, + ) class CreateBackupRequest(proto.Message): @@ -204,6 +228,91 @@ class CreateBackupMetadata(proto.Message): cancel_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,) +class CopyBackupRequest(proto.Message): + r"""The request for + [CopyBackup][google.spanner.admin.database.v1.DatabaseAdmin.CopyBackup]. + + Attributes: + parent (str): + Required. The name of the destination instance that will + contain the backup copy. Values are of the form: + ``projects//instances/``. + backup_id (str): + Required. The id of the backup copy. The ``backup_id`` + appended to ``parent`` forms the full backup_uri of the form + ``projects//instances//backups/``. + source_backup (str): + Required. The source backup to be copied. The source backup + needs to be in READY state for it to be copied. Once + CopyBackup is in progress, the source backup cannot be + deleted or cleaned up on expiration until CopyBackup is + finished. Values are of the form: + ``projects//instances//backups/``. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Required. The expiration time of the backup in microsecond + granularity. The expiration time must be at least 6 hours + and at most 366 days from the ``create_time`` of the source + backup. Once the ``expire_time`` has passed, the backup is + eligible to be automatically deleted by Cloud Spanner to + free the resources used by the backup. + encryption_config (google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig): + Optional. The encryption configuration used to encrypt the + backup. If this field is not specified, the backup will use + the same encryption configuration as the source backup by + default, namely + [encryption_type][google.spanner.admin.database.v1.CopyBackupEncryptionConfig.encryption_type] + = ``USE_CONFIG_DEFAULT_OR_BACKUP_ENCRYPTION``. + """ + + parent = proto.Field(proto.STRING, number=1,) + backup_id = proto.Field(proto.STRING, number=2,) + source_backup = proto.Field(proto.STRING, number=3,) + expire_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,) + encryption_config = proto.Field( + proto.MESSAGE, number=5, message="CopyBackupEncryptionConfig", + ) + + +class CopyBackupMetadata(proto.Message): + r"""Metadata type for the google.longrunning.Operation returned by + [CopyBackup][google.spanner.admin.database.v1.DatabaseAdmin.CopyBackup]. + + Attributes: + name (str): + The name of the backup being created through the copy + operation. Values are of the form + ``projects//instances//backups/``. + source_backup (str): + The name of the source backup that is being copied. Values + are of the form + ``projects//instances//backups/``. + progress (google.cloud.spanner_admin_database_v1.types.OperationProgress): + The progress of the + [CopyBackup][google.spanner.admin.database.v1.DatabaseAdmin.CopyBackup] + operation. + cancel_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which cancellation of CopyBackup operation was + received. + [Operations.CancelOperation][google.longrunning.Operations.CancelOperation] + starts asynchronous cancellation on a long-running + operation. The server makes a best effort to cancel the + operation, but success is not guaranteed. Clients can use + [Operations.GetOperation][google.longrunning.Operations.GetOperation] + or other methods to check whether the cancellation succeeded + or whether the operation completed despite cancellation. On + successful cancellation, the operation is not deleted; + instead, it becomes an operation with an + [Operation.error][google.longrunning.Operation.error] value + with a [google.rpc.Status.code][google.rpc.Status.code] of + 1, corresponding to ``Code.CANCELLED``. + """ + + name = proto.Field(proto.STRING, number=1,) + source_backup = proto.Field(proto.STRING, number=2,) + progress = proto.Field(proto.MESSAGE, number=3, message=common.OperationProgress,) + cancel_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,) + + class UpdateBackupRequest(proto.Message): r"""The request for [UpdateBackup][google.spanner.admin.database.v1.DatabaseAdmin.UpdateBackup]. @@ -386,6 +495,8 @@ class ListBackupOperationsRequest(proto.Message): is ``type.googleapis.com/google.spanner.admin.database.v1.CreateBackupMetadata``. - ``metadata.`` - any field in metadata.value. + ``metadata.@type`` must be specified first if filtering + on metadata fields. - ``error`` - Error associated with the long-running operation. - ``response.@type`` - the type of response. @@ -399,8 +510,14 @@ class ListBackupOperationsRequest(proto.Message): Here are a few examples: - ``done:true`` - The operation is complete. - - ``metadata.database:prod`` - The database the backup was - taken from has a name containing the string "prod". + - ``(metadata.@type=type.googleapis.com/google.spanner.admin.database.v1.CreateBackupMetadata) AND`` + ``metadata.database:prod`` - Returns operations where: + + - The operation's metadata type is + [CreateBackupMetadata][google.spanner.admin.database.v1.CreateBackupMetadata]. + - The database the backup was taken from has a name + containing the string "prod". + - ``(metadata.@type=type.googleapis.com/google.spanner.admin.database.v1.CreateBackupMetadata) AND`` ``(metadata.name:howl) AND`` ``(metadata.progress.start_time < \"2018-03-28T14:50:00Z\") AND`` @@ -411,6 +528,37 @@ class ListBackupOperationsRequest(proto.Message): - The backup name contains the string "howl". - The operation started before 2018-03-28T14:50:00Z. - The operation resulted in an error. + + - ``(metadata.@type=type.googleapis.com/google.spanner.admin.database.v1.CopyBackupMetadata) AND`` + ``(metadata.source_backup:test) AND`` + ``(metadata.progress.start_time < \"2022-01-18T14:50:00Z\") AND`` + ``(error:*)`` - Returns operations where: + + - The operation's metadata type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata]. + - The source backup of the copied backup name contains + the string "test". + - The operation started before 2022-01-18T14:50:00Z. + - The operation resulted in an error. + + - ``((metadata.@type=type.googleapis.com/google.spanner.admin.database.v1.CreateBackupMetadata) AND`` + ``(metadata.database:test_db)) OR`` + ``((metadata.@type=type.googleapis.com/google.spanner.admin.database.v1.CopyBackupMetadata) AND`` + ``(metadata.source_backup:test_bkp)) AND`` + ``(error:*)`` - Returns operations where: + + - The operation's metadata matches either of criteria: + + - The operation's metadata type is + [CreateBackupMetadata][google.spanner.admin.database.v1.CreateBackupMetadata] + AND the database the backup was taken from has name + containing string "test_db" + - The operation's metadata type is + [CopyBackupMetadata][google.spanner.admin.database.v1.CopyBackupMetadata] + AND the backup the backup was copied from has name + containing string "test_bkp" + + - The operation resulted in an error. page_size (int): Number of operations to be returned in the response. If 0 or less, defaults to the server's @@ -437,11 +585,9 @@ class ListBackupOperationsResponse(proto.Message): operations (Sequence[google.longrunning.operations_pb2.Operation]): The list of matching backup [long-running operations][google.longrunning.Operation]. Each operation's - name will be prefixed by the backup's name and the - operation's - [metadata][google.longrunning.Operation.metadata] will be of - type - [CreateBackupMetadata][google.spanner.admin.database.v1.CreateBackupMetadata]. + name will be prefixed by the backup's name. The operation's + [metadata][google.longrunning.Operation.metadata] field type + ``metadata.type_url`` describes the type of the metadata. Operations returned include those that are pending or have completed/failed/canceled within the last 7 days. Operations returned are ordered by @@ -520,4 +666,29 @@ class EncryptionType(proto.Enum): kms_key_name = proto.Field(proto.STRING, number=2,) +class CopyBackupEncryptionConfig(proto.Message): + r"""Encryption configuration for the copied backup. + + Attributes: + encryption_type (google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig.EncryptionType): + Required. The encryption type of the backup. + kms_key_name (str): + Optional. The Cloud KMS key that will be used to protect the + backup. This field should be set only when + [encryption_type][google.spanner.admin.database.v1.CopyBackupEncryptionConfig.encryption_type] + is ``CUSTOMER_MANAGED_ENCRYPTION``. Values are of the form + ``projects//locations//keyRings//cryptoKeys/``. + """ + + class EncryptionType(proto.Enum): + r"""Encryption types for the backup.""" + ENCRYPTION_TYPE_UNSPECIFIED = 0 + USE_CONFIG_DEFAULT_OR_BACKUP_ENCRYPTION = 1 + GOOGLE_DEFAULT_ENCRYPTION = 2 + CUSTOMER_MANAGED_ENCRYPTION = 3 + + encryption_type = proto.Field(proto.ENUM, number=1, enum=EncryptionType,) + kms_key_name = proto.Field(proto.STRING, number=2,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/spanner_admin_database_v1/types/spanner_database_admin.py b/google/cloud/spanner_admin_database_v1/types/spanner_database_admin.py index 42cf4f484f..c9c519334b 100644 --- a/google/cloud/spanner_admin_database_v1/types/spanner_database_admin.py +++ b/google/cloud/spanner_admin_database_v1/types/spanner_database_admin.py @@ -447,6 +447,8 @@ class ListDatabaseOperationsRequest(proto.Message): is ``type.googleapis.com/google.spanner.admin.database.v1.RestoreDatabaseMetadata``. - ``metadata.`` - any field in metadata.value. + ``metadata.@type`` must be specified first, if filtering + on metadata fields. - ``error`` - Error associated with the long-running operation. - ``response.@type`` - the type of response. diff --git a/samples/generated_samples/snippet_metadata_spanner admin database_v1.json b/samples/generated_samples/snippet_metadata_spanner admin database_v1.json index 10a85bf3f2..5564ff3d37 100644 --- a/samples/generated_samples/snippet_metadata_spanner admin database_v1.json +++ b/samples/generated_samples/snippet_metadata_spanner admin database_v1.json @@ -1,5 +1,94 @@ { "snippets": [ + { + "clientMethod": { + "async": true, + "method": { + "service": { + "shortName": "DatabaseAdmin" + }, + "shortName": "CopyBackup" + } + }, + "file": "spanner_v1_generated_database_admin_copy_backup_async.py", + "regionTag": "spanner_v1_generated_DatabaseAdmin_CopyBackup_async", + "segments": [ + { + "end": 50, + "start": 27, + "type": "FULL" + }, + { + "end": 50, + "start": 27, + "type": "SHORT" + }, + { + "end": 33, + "start": 31, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 40, + "start": 34, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 47, + "start": 41, + "type": "REQUEST_EXECUTION" + }, + { + "end": 51, + "start": 48, + "type": "RESPONSE_HANDLING" + } + ] + }, + { + "clientMethod": { + "method": { + "service": { + "shortName": "DatabaseAdmin" + }, + "shortName": "CopyBackup" + } + }, + "file": "spanner_v1_generated_database_admin_copy_backup_sync.py", + "regionTag": "spanner_v1_generated_DatabaseAdmin_CopyBackup_sync", + "segments": [ + { + "end": 50, + "start": 27, + "type": "FULL" + }, + { + "end": 50, + "start": 27, + "type": "SHORT" + }, + { + "end": 33, + "start": 31, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 40, + "start": 34, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 47, + "start": 41, + "type": "REQUEST_EXECUTION" + }, + { + "end": 51, + "start": 48, + "type": "RESPONSE_HANDLING" + } + ] + }, { "clientMethod": { "async": true, diff --git a/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_async.py b/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_async.py new file mode 100644 index 0000000000..645e606faf --- /dev/null +++ b/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_async.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CopyBackup +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-spanner-admin-database + + +# [START spanner_v1_generated_DatabaseAdmin_CopyBackup_async] +from google.cloud import spanner_admin_database_v1 + + +async def sample_copy_backup(): + # Create a client + client = spanner_admin_database_v1.DatabaseAdminAsyncClient() + + # Initialize request argument(s) + request = spanner_admin_database_v1.CopyBackupRequest( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + ) + + # Make the request + operation = client.copy_backup(request=request) + + print("Waiting for operation to complete...") + + response = await operation.result() + + # Handle the response + print(response) + +# [END spanner_v1_generated_DatabaseAdmin_CopyBackup_async] diff --git a/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_sync.py b/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_sync.py new file mode 100644 index 0000000000..f5babd289c --- /dev/null +++ b/samples/generated_samples/spanner_v1_generated_database_admin_copy_backup_sync.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CopyBackup +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-spanner-admin-database + + +# [START spanner_v1_generated_DatabaseAdmin_CopyBackup_sync] +from google.cloud import spanner_admin_database_v1 + + +def sample_copy_backup(): + # Create a client + client = spanner_admin_database_v1.DatabaseAdminClient() + + # Initialize request argument(s) + request = spanner_admin_database_v1.CopyBackupRequest( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + ) + + # Make the request + operation = client.copy_backup(request=request) + + print("Waiting for operation to complete...") + + response = operation.result() + + # Handle the response + print(response) + +# [END spanner_v1_generated_DatabaseAdmin_CopyBackup_sync] diff --git a/scripts/fixup_spanner_admin_database_v1_keywords.py b/scripts/fixup_spanner_admin_database_v1_keywords.py index 5a0630802f..5c11670473 100644 --- a/scripts/fixup_spanner_admin_database_v1_keywords.py +++ b/scripts/fixup_spanner_admin_database_v1_keywords.py @@ -39,6 +39,7 @@ def partition( class spanner_admin_databaseCallTransformer(cst.CSTTransformer): CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'copy_backup': ('parent', 'backup_id', 'source_backup', 'expire_time', 'encryption_config', ), 'create_backup': ('parent', 'backup_id', 'backup', 'encryption_config', ), 'create_database': ('parent', 'create_statement', 'extra_statements', 'encryption_config', 'database_dialect', ), 'delete_backup': ('name', ), diff --git a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py index de918f8c79..71fb398101 100644 --- a/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py +++ b/tests/unit/gapic/spanner_admin_database_v1/test_database_admin.py @@ -3005,6 +3005,241 @@ async def test_create_backup_flattened_error_async(): ) +@pytest.mark.parametrize("request_type", [backup.CopyBackupRequest, dict,]) +def test_copy_backup(request_type, transport: str = "grpc"): + client = DatabaseAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/spam") + response = client.copy_backup(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == backup.CopyBackupRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +def test_copy_backup_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatabaseAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + client.copy_backup() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == backup.CopyBackupRequest() + + +@pytest.mark.asyncio +async def test_copy_backup_async( + transport: str = "grpc_asyncio", request_type=backup.CopyBackupRequest +): + client = DatabaseAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + response = await client.copy_backup(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == backup.CopyBackupRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +@pytest.mark.asyncio +async def test_copy_backup_async_from_dict(): + await test_copy_backup_async(request_type=dict) + + +def test_copy_backup_field_headers(): + client = DatabaseAdminClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = backup.CopyBackupRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + call.return_value = operations_pb2.Operation(name="operations/op") + client.copy_backup(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +@pytest.mark.asyncio +async def test_copy_backup_field_headers_async(): + client = DatabaseAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = backup.CopyBackupRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/op") + ) + await client.copy_backup(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +def test_copy_backup_flattened(): + client = DatabaseAdminClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/op") + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.copy_backup( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + expire_time=timestamp_pb2.Timestamp(seconds=751), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = "parent_value" + assert arg == mock_val + arg = args[0].backup_id + mock_val = "backup_id_value" + assert arg == mock_val + arg = args[0].source_backup + mock_val = "source_backup_value" + assert arg == mock_val + assert TimestampRule().to_proto(args[0].expire_time) == timestamp_pb2.Timestamp( + seconds=751 + ) + + +def test_copy_backup_flattened_error(): + client = DatabaseAdminClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.copy_backup( + backup.CopyBackupRequest(), + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + expire_time=timestamp_pb2.Timestamp(seconds=751), + ) + + +@pytest.mark.asyncio +async def test_copy_backup_flattened_async(): + client = DatabaseAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.copy_backup), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/op") + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.copy_backup( + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + expire_time=timestamp_pb2.Timestamp(seconds=751), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = "parent_value" + assert arg == mock_val + arg = args[0].backup_id + mock_val = "backup_id_value" + assert arg == mock_val + arg = args[0].source_backup + mock_val = "source_backup_value" + assert arg == mock_val + assert TimestampRule().to_proto(args[0].expire_time) == timestamp_pb2.Timestamp( + seconds=751 + ) + + +@pytest.mark.asyncio +async def test_copy_backup_flattened_error_async(): + client = DatabaseAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.copy_backup( + backup.CopyBackupRequest(), + parent="parent_value", + backup_id="backup_id_value", + source_backup="source_backup_value", + expire_time=timestamp_pb2.Timestamp(seconds=751), + ) + + @pytest.mark.parametrize("request_type", [backup.GetBackupRequest, dict,]) def test_get_backup(request_type, transport: str = "grpc"): client = DatabaseAdminClient( @@ -3025,6 +3260,7 @@ def test_get_backup(request_type, transport: str = "grpc"): state=backup.Backup.State.CREATING, referencing_databases=["referencing_databases_value"], database_dialect=common.DatabaseDialect.GOOGLE_STANDARD_SQL, + referencing_backups=["referencing_backups_value"], ) response = client.get_backup(request) @@ -3041,6 +3277,7 @@ def test_get_backup(request_type, transport: str = "grpc"): assert response.state == backup.Backup.State.CREATING assert response.referencing_databases == ["referencing_databases_value"] assert response.database_dialect == common.DatabaseDialect.GOOGLE_STANDARD_SQL + assert response.referencing_backups == ["referencing_backups_value"] def test_get_backup_empty_call(): @@ -3081,6 +3318,7 @@ async def test_get_backup_async( state=backup.Backup.State.CREATING, referencing_databases=["referencing_databases_value"], database_dialect=common.DatabaseDialect.GOOGLE_STANDARD_SQL, + referencing_backups=["referencing_backups_value"], ) ) response = await client.get_backup(request) @@ -3098,6 +3336,7 @@ async def test_get_backup_async( assert response.state == backup.Backup.State.CREATING assert response.referencing_databases == ["referencing_databases_value"] assert response.database_dialect == common.DatabaseDialect.GOOGLE_STANDARD_SQL + assert response.referencing_backups == ["referencing_backups_value"] @pytest.mark.asyncio @@ -3246,6 +3485,7 @@ def test_update_backup(request_type, transport: str = "grpc"): state=gsad_backup.Backup.State.CREATING, referencing_databases=["referencing_databases_value"], database_dialect=common.DatabaseDialect.GOOGLE_STANDARD_SQL, + referencing_backups=["referencing_backups_value"], ) response = client.update_backup(request) @@ -3262,6 +3502,7 @@ def test_update_backup(request_type, transport: str = "grpc"): assert response.state == gsad_backup.Backup.State.CREATING assert response.referencing_databases == ["referencing_databases_value"] assert response.database_dialect == common.DatabaseDialect.GOOGLE_STANDARD_SQL + assert response.referencing_backups == ["referencing_backups_value"] def test_update_backup_empty_call(): @@ -3302,6 +3543,7 @@ async def test_update_backup_async( state=gsad_backup.Backup.State.CREATING, referencing_databases=["referencing_databases_value"], database_dialect=common.DatabaseDialect.GOOGLE_STANDARD_SQL, + referencing_backups=["referencing_backups_value"], ) ) response = await client.update_backup(request) @@ -3319,6 +3561,7 @@ async def test_update_backup_async( assert response.state == gsad_backup.Backup.State.CREATING assert response.referencing_databases == ["referencing_databases_value"] assert response.database_dialect == common.DatabaseDialect.GOOGLE_STANDARD_SQL + assert response.referencing_backups == ["referencing_backups_value"] @pytest.mark.asyncio @@ -5076,6 +5319,7 @@ def test_database_admin_base_transport(): "get_iam_policy", "test_iam_permissions", "create_backup", + "copy_backup", "get_backup", "update_backup", "delete_backup", From caf4b0ba04f5c94f9a26e9d748ddc7e623c6c8db Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Wed, 23 Mar 2022 16:54:56 +0530 Subject: [PATCH 10/11] feat: adding samples --- google/cloud/spanner_v1/backup.py | 35 ++++++++++++--------------- google/cloud/spanner_v1/instance.py | 2 ++ samples/samples/backup_sample.py | 7 +++--- samples/samples/backup_sample_test.py | 23 +++++++++--------- tests/system/_helpers.py | 0 tests/system/conftest.py | 3 --- tests/system/test_backup_api.py | 20 --------------- 7 files changed, 33 insertions(+), 57 deletions(-) mode change 100755 => 100644 google/cloud/spanner_v1/backup.py mode change 100755 => 100644 google/cloud/spanner_v1/instance.py mode change 100755 => 100644 tests/system/_helpers.py mode change 100755 => 100644 tests/system/conftest.py mode change 100755 => 100644 tests/system/test_backup_api.py diff --git a/google/cloud/spanner_v1/backup.py b/google/cloud/spanner_v1/backup.py old mode 100755 new mode 100644 index 1abf9d1df4..d7a97809f1 --- a/google/cloud/spanner_v1/backup.py +++ b/google/cloud/spanner_v1/backup.py @@ -266,20 +266,27 @@ def create(self): if not self._expire_time: raise ValueError("expire_time not set") - api = self._instance._client.database_admin_api - metadata = _metadata_with_prefix(self.name) + if not self._database and not self._source_backup: + raise ValueError("database and source backup both not set") - if self._source_backup: - if ( + if ( + ( self._encryption_config and self._encryption_config.kms_key_name and self._encryption_config.encryption_type - != CopyBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION - ): - raise ValueError( - "kms_key_name only used with CUSTOMER_MANAGED_ENCRYPTION" - ) + != CreateBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION + ) + and self._encryption_config + and self._encryption_config.kms_key_name + and self._encryption_config.encryption_type + != CopyBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION + ): + raise ValueError("kms_key_name only used with CUSTOMER_MANAGED_ENCRYPTION") + api = self._instance._client.database_admin_api + metadata = _metadata_with_prefix(self.name) + + if self._source_backup: request = CopyBackupRequest( parent=self._instance.name, backup_id=self.backup_id, @@ -291,16 +298,6 @@ def create(self): future = api.copy_backup(request=request, metadata=metadata,) return future - if not self._database: - raise ValueError("database not set") - if ( - self._encryption_config - and self._encryption_config.kms_key_name - and self._encryption_config.encryption_type - != CreateBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION - ): - raise ValueError("kms_key_name only used with CUSTOMER_MANAGED_ENCRYPTION") - backup = BackupPB( database=self._database, expire_time=self.expire_time, diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py old mode 100755 new mode 100644 index 454c8c72b2..a7725c7057 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -44,6 +44,7 @@ _OPERATION_METADATA_MESSAGES = ( backup.Backup, backup.CreateBackupMetadata, + backup.CopyBackupMetadata, spanner_database_admin.CreateDatabaseMetadata, spanner_database_admin.Database, spanner_database_admin.OptimizeRestoredDatabaseMetadata, @@ -58,6 +59,7 @@ _OPERATION_RESPONSE_TYPES = { backup.CreateBackupMetadata: backup.Backup, + backup.CopyBackupMetadata: backup.Backup, spanner_database_admin.CreateDatabaseMetadata: spanner_database_admin.Database, spanner_database_admin.OptimizeRestoredDatabaseMetadata: spanner_database_admin.Database, spanner_database_admin.RestoreDatabaseMetadata: spanner_database_admin.Database, diff --git a/samples/samples/backup_sample.py b/samples/samples/backup_sample.py index 66ab8d3e19..b2f63ca29e 100644 --- a/samples/samples/backup_sample.py +++ b/samples/samples/backup_sample.py @@ -213,9 +213,8 @@ def list_backup_operations(instance_id, database_id, backup_id): # List the CopyBackup operations. filter_ = ( - "(metadata.@type:type.googleapis.com/" - "google.spanner.admin.database.v1.CopyBackupMetadata)" - " AND (metadata.source_backup:{})" + "(metadata.@type:type.googleapis.com/google.spanner.admin.database.v1.CopyBackupMetadata) " + "AND (metadata.source_backup:{})" ).format(backup_id) operations = instance.list_backup_operations(filter_=filter_) for op in operations: @@ -405,7 +404,7 @@ def copy_backup(instance_id, backup_id, source_backup_path): # Create a backup object and wait for copy backup operation to complete. expire_time = datetime.utcnow() + timedelta(days=14) - copy_backup = instance.copy_backup(backup_id, source_backup=source_backup_path, expire_time=expire_time) + copy_backup = instance.copy_backup(backup_id=backup_id, source_backup=source_backup_path, expire_time=expire_time) operation = copy_backup.create() # Wait for copy backup operation to complete. diff --git a/samples/samples/backup_sample_test.py b/samples/samples/backup_sample_test.py index f53499c362..ffecdd814b 100644 --- a/samples/samples/backup_sample_test.py +++ b/samples/samples/backup_sample_test.py @@ -61,6 +61,18 @@ def test_create_backup(capsys, instance_id, sample_database): assert BACKUP_ID in out +@pytest.mark.dependency(name="copy_backup",depends=["create_backup"]) +def test_copy_backup(capsys, instance_id, spanner_client): + source_backp_path=spanner_client.project_name+'/instances/'+instance_id+'/backups/'+BACKUP_ID + backup_sample.copy_backup( + instance_id, + COPY_BACKUP_ID, + source_backp_path + ) + out, _ = capsys.readouterr() + assert COPY_BACKUP_ID in out + + @pytest.mark.dependency(name="create_backup_with_encryption_key") def test_create_backup_with_encryption_key( capsys, instance_id, sample_database, kms_key_name, @@ -162,14 +174,3 @@ def test_create_database_with_retention_period(capsys, sample_instance): database = sample_instance.database(RETENTION_DATABASE_ID) database.drop() -@pytest.mark.dependency(name="copy_backup",depends=["create_backup"]) -def test_copy_backup(capsys, instance_id, spanner_client): - source_backp_path=spanner_client.project_name+'/instances/'+instance_id+'/backups/'+BACKUP_ID - backup_sample.copy_backup( - instance_id, - source_backp_path, - BACKUP_ID - ) - out, _ = capsys.readouterr() - assert COPY_BACKUP_ID in out - diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py old mode 100755 new mode 100644 diff --git a/tests/system/conftest.py b/tests/system/conftest.py old mode 100755 new mode 100644 index 868c209151..40b76208e8 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -14,9 +14,6 @@ import datetime import time -from google.cloud.spanner_admin_database_v1.types.backup import ( - CreateBackupEncryptionConfig, -) import pytest diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py old mode 100755 new mode 100644 index 97585bef1f..f7325dc356 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -203,7 +203,6 @@ def test_copy_backup_workflow( shared_instance, shared_backup, backups_to_delete, ): from google.cloud.spanner_admin_database_v1 import ( - CreateBackupEncryptionConfig, CopyBackupEncryptionConfig, EncryptionInfo, ) @@ -346,25 +345,6 @@ def test_backup_create_w_invalid_version_time_future( op.result() # blocks indefinitely -def test_copy_backup_create_w_invalid_expire_time( - shared_instance, shared_backup, -): - backup_id = _helpers.unique_id("backup_id", separator="_") - invalid_expire_time = datetime.datetime.now(datetime.timezone.utc) - - shared_backup.reload() - - copy_backup = shared_instance.copy_backup( - backup_id=backup_id, - source_backup=shared_backup.name, - expire_time=invalid_expire_time, - ) - - with pytest.raises(exceptions.InvalidArgument): - operation = copy_backup.create() - operation.result() # blocks indefinitely - - def test_database_restore_to_diff_instance( shared_instance, shared_database, From ac4e13bf94ed2d722eb607555a5c036b9c139c8b Mon Sep 17 00:00:00 2001 From: Astha Mohta Date: Thu, 24 Mar 2022 10:29:00 +0530 Subject: [PATCH 11/11] linting --- google/cloud/spanner_v1/instance.py | 37 ++++++------ samples/samples/autocommit.py | 7 +-- samples/samples/autocommit_test.py | 2 +- samples/samples/backup_sample.py | 69 ++++++++++++++------- samples/samples/backup_sample_test.py | 39 ++++++------ samples/samples/conftest.py | 6 +- samples/samples/noxfile.py | 8 +-- samples/samples/snippets.py | 86 ++++++++++++++------------- samples/samples/snippets_test.py | 36 +++++++++-- 9 files changed, 171 insertions(+), 119 deletions(-) diff --git a/google/cloud/spanner_v1/instance.py b/google/cloud/spanner_v1/instance.py index a7725c7057..d3514bd85d 100644 --- a/google/cloud/spanner_v1/instance.py +++ b/google/cloud/spanner_v1/instance.py @@ -557,24 +557,25 @@ def copy_backup( self, backup_id, source_backup, expire_time=None, encryption_config=None, ): """Factory to create a copy backup within this instance. - :type backup_id: str - :param backup_id: The ID of the backup copy. - :type source_backup: str - :param source_backup_id: The full path of the source backup to be copied. - :type expire_time: :class:`datetime.datetime` - :param expire_time: - Optional. The expire time that will be used when creating the copy backup. - Required if the create method needs to be called. - :type encryption_config: - :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` - or :class:`dict` - :param encryption_config: - (Optional) Encryption configuration for the backup. - If a dict is provided, it must be of the same form as the protobuf - message :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` - :rtype: :class:`~google.cloud.spanner_v1.backup.Backup` - :returns: a copy backup owned by this instance. - """ + + :type backup_id: str + :param backup_id: The ID of the backup copy. + :type source_backup: str + :param source_backup_id: The full path of the source backup to be copied. + :type expire_time: :class:`datetime.datetime` + :param expire_time: + Optional. The expire time that will be used when creating the copy backup. + Required if the create method needs to be called. + :type encryption_config: + :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` + or :class:`dict` + :param encryption_config: + (Optional) Encryption configuration for the backup. + If a dict is provided, it must be of the same form as the protobuf + message :class:`~google.cloud.spanner_admin_database_v1.types.CopyBackupEncryptionConfig` + :rtype: :class:`~google.cloud.spanner_v1.backup.Backup` + :returns: a copy backup owned by this instance. + """ return Backup( backup_id, self, diff --git a/samples/samples/autocommit.py b/samples/samples/autocommit.py index 873ed2b7bd..d5c44b0c53 100644 --- a/samples/samples/autocommit.py +++ b/samples/samples/autocommit.py @@ -46,14 +46,11 @@ def enable_autocommit_mode(instance_id, database_id): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("instance_id", help="Your Cloud Spanner instance ID.") parser.add_argument( - "--database-id", - help="Your Cloud Spanner database ID.", - default="example_db", + "--database-id", help="Your Cloud Spanner database ID.", default="example_db", ) subparsers = parser.add_subparsers(dest="command") subparsers.add_parser("enable_autocommit_mode", help=enable_autocommit_mode.__doc__) diff --git a/samples/samples/autocommit_test.py b/samples/samples/autocommit_test.py index 9880460cac..6b102da8fe 100644 --- a/samples/samples/autocommit_test.py +++ b/samples/samples/autocommit_test.py @@ -19,7 +19,7 @@ def sample_name(): @RetryErrors(exception=Aborted, max_tries=2) def test_enable_autocommit_mode(capsys, instance_id, sample_database): # Delete table if it exists for retry attempts. - table = sample_database.table('Singers') + table = sample_database.table("Singers") if table.exists(): op = sample_database.update_ddl(["DROP TABLE Singers"]) op.result() diff --git a/samples/samples/backup_sample.py b/samples/samples/backup_sample.py index b2f63ca29e..01d3e4bf60 100644 --- a/samples/samples/backup_sample.py +++ b/samples/samples/backup_sample.py @@ -34,7 +34,9 @@ def create_backup(instance_id, database_id, backup_id, version_time): # Create a backup expire_time = datetime.utcnow() + timedelta(days=14) - backup = instance.backup(backup_id, database=database, expire_time=expire_time, version_time=version_time) + backup = instance.backup( + backup_id, database=database, expire_time=expire_time, version_time=version_time + ) operation = backup.create() # Wait for backup operation to complete. @@ -56,7 +58,9 @@ def create_backup(instance_id, database_id, backup_id, version_time): # [END spanner_create_backup] # [START spanner_create_backup_with_encryption_key] -def create_backup_with_encryption_key(instance_id, database_id, backup_id, kms_key_name): +def create_backup_with_encryption_key( + instance_id, database_id, backup_id, kms_key_name +): """Creates a backup for a database using a Customer Managed Encryption Key (CMEK).""" from google.cloud.spanner_admin_database_v1 import CreateBackupEncryptionConfig @@ -67,10 +71,15 @@ def create_backup_with_encryption_key(instance_id, database_id, backup_id, kms_k # Create a backup expire_time = datetime.utcnow() + timedelta(days=14) encryption_config = { - 'encryption_type': CreateBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION, - 'kms_key_name': kms_key_name, + "encryption_type": CreateBackupEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION, + "kms_key_name": kms_key_name, } - backup = instance.backup(backup_id, database=database, expire_time=expire_time, encryption_config=encryption_config) + backup = instance.backup( + backup_id, + database=database, + expire_time=expire_time, + encryption_config=encryption_config, + ) operation = backup.create() # Wait for backup operation to complete. @@ -115,7 +124,7 @@ def restore_database(instance_id, new_database_id, backup_id): restore_info.backup_info.source_database, new_database_id, restore_info.backup_info.backup, - restore_info.backup_info.version_time + restore_info.backup_info.version_time, ) ) @@ -124,7 +133,9 @@ def restore_database(instance_id, new_database_id, backup_id): # [START spanner_restore_backup_with_encryption_key] -def restore_database_with_encryption_key(instance_id, new_database_id, backup_id, kms_key_name): +def restore_database_with_encryption_key( + instance_id, new_database_id, backup_id, kms_key_name +): """Restores a database from a backup using a Customer Managed Encryption Key (CMEK).""" from google.cloud.spanner_admin_database_v1 import RestoreDatabaseEncryptionConfig @@ -134,10 +145,12 @@ def restore_database_with_encryption_key(instance_id, new_database_id, backup_id # Start restoring an existing backup to a new database. backup = instance.backup(backup_id) encryption_config = { - 'encryption_type': RestoreDatabaseEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION, - 'kms_key_name': kms_key_name, + "encryption_type": RestoreDatabaseEncryptionConfig.EncryptionType.CUSTOMER_MANAGED_ENCRYPTION, + "kms_key_name": kms_key_name, } - new_database = instance.database(new_database_id, encryption_config=encryption_config) + new_database = instance.database( + new_database_id, encryption_config=encryption_config + ) operation = new_database.restore(backup) # Wait for restore operation to complete. @@ -210,7 +223,7 @@ def list_backup_operations(instance_id, database_id, backup_id): metadata.name, metadata.database, metadata.progress.progress_percent ) ) - + # List the CopyBackup operations. filter_ = ( "(metadata.@type:type.googleapis.com/google.spanner.admin.database.v1.CopyBackupMetadata) " @@ -221,7 +234,9 @@ def list_backup_operations(instance_id, database_id, backup_id): metadata = op.metadata print( "Backup {} on source backup {}: {}% complete.".format( - metadata.name, metadata.source_backup, metadata.progress.progress_percent + metadata.name, + metadata.source_backup, + metadata.progress.progress_percent, ) ) @@ -305,8 +320,11 @@ def list_backups(instance_id, database_id, backup_id): print("All backups with pagination") # If there are multiple pages, additional ``ListBackup`` # requests will be made as needed while iterating. + paged_backups = set() for backup in instance.list_backups(page_size=2): - print(backup.name) + paged_backups.add(backup.name) + for backup in paged_backups: + print(backup) # [END spanner_list_backups] @@ -358,7 +376,9 @@ def update_backup(instance_id, backup_id): # [START spanner_create_database_with_version_retention_period] -def create_database_with_version_retention_period(instance_id, database_id, retention_period): +def create_database_with_version_retention_period( + instance_id, database_id, retention_period +): """Creates a database with a version retention period.""" spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) @@ -378,7 +398,7 @@ def create_database_with_version_retention_period(instance_id, database_id, rete "ALTER DATABASE `{}`" " SET OPTIONS (version_retention_period = '{}')".format( database_id, retention_period - ) + ), ] db = instance.database(database_id, ddl_statements) operation = db.create() @@ -387,12 +407,15 @@ def create_database_with_version_retention_period(instance_id, database_id, rete db.reload() - print("Database {} created with version retention period {} and earliest version time {}".format( - db.database_id, db.version_retention_period, db.earliest_version_time - )) + print( + "Database {} created with version retention period {} and earliest version time {}".format( + db.database_id, db.version_retention_period, db.earliest_version_time + ) + ) db.drop() + # [END spanner_create_database_with_version_retention_period] @@ -404,7 +427,9 @@ def copy_backup(instance_id, backup_id, source_backup_path): # Create a backup object and wait for copy backup operation to complete. expire_time = datetime.utcnow() + timedelta(days=14) - copy_backup = instance.copy_backup(backup_id=backup_id, source_backup=source_backup_path, expire_time=expire_time) + copy_backup = instance.copy_backup( + backup_id=backup_id, source_backup=source_backup_path, expire_time=expire_time + ) operation = copy_backup.create() # Wait for copy backup operation to complete. @@ -416,10 +441,14 @@ def copy_backup(instance_id, backup_id, source_backup_path): print( "Backup {} of size {} bytes was created at {} with version time {}".format( - copy_backup.name, copy_backup.size_bytes, copy_backup.create_time, copy_backup.version_time, + copy_backup.name, + copy_backup.size_bytes, + copy_backup.create_time, + copy_backup.version_time, ) ) + # [END spanner_copy_backup] diff --git a/samples/samples/backup_sample_test.py b/samples/samples/backup_sample_test.py index ffecdd814b..da50fbba46 100644 --- a/samples/samples/backup_sample_test.py +++ b/samples/samples/backup_sample_test.py @@ -52,23 +52,22 @@ def test_create_backup(capsys, instance_id, sample_database): version_time = list(results)[0][0] backup_sample.create_backup( - instance_id, - sample_database.database_id, - BACKUP_ID, - version_time, + instance_id, sample_database.database_id, BACKUP_ID, version_time, ) out, _ = capsys.readouterr() assert BACKUP_ID in out -@pytest.mark.dependency(name="copy_backup",depends=["create_backup"]) +@pytest.mark.dependency(name="copy_backup", depends=["create_backup"]) def test_copy_backup(capsys, instance_id, spanner_client): - source_backp_path=spanner_client.project_name+'/instances/'+instance_id+'/backups/'+BACKUP_ID - backup_sample.copy_backup( - instance_id, - COPY_BACKUP_ID, - source_backp_path + source_backp_path = ( + spanner_client.project_name + + "/instances/" + + instance_id + + "/backups/" + + BACKUP_ID ) + backup_sample.copy_backup(instance_id, COPY_BACKUP_ID, source_backp_path) out, _ = capsys.readouterr() assert COPY_BACKUP_ID in out @@ -78,10 +77,7 @@ def test_create_backup_with_encryption_key( capsys, instance_id, sample_database, kms_key_name, ): backup_sample.create_backup_with_encryption_key( - instance_id, - sample_database.database_id, - CMEK_BACKUP_ID, - kms_key_name, + instance_id, sample_database.database_id, CMEK_BACKUP_ID, kms_key_name, ) out, _ = capsys.readouterr() assert CMEK_BACKUP_ID in out @@ -104,7 +100,8 @@ def test_restore_database_with_encryption_key( capsys, instance_id, sample_database, kms_key_name, ): backup_sample.restore_database_with_encryption_key( - instance_id, CMEK_RESTORE_DB_ID, CMEK_BACKUP_ID, kms_key_name) + instance_id, CMEK_RESTORE_DB_ID, CMEK_BACKUP_ID, kms_key_name + ) out, _ = capsys.readouterr() assert (sample_database.database_id + " restored to ") in out assert (CMEK_RESTORE_DB_ID + " from backup ") in out @@ -115,7 +112,8 @@ def test_restore_database_with_encryption_key( @pytest.mark.dependency(depends=["create_backup", "copy_backup"]) def test_list_backup_operations(capsys, instance_id, sample_database): backup_sample.list_backup_operations( - instance_id, sample_database.database_id, BACKUP_ID) + instance_id, sample_database.database_id, BACKUP_ID + ) out, _ = capsys.readouterr() assert BACKUP_ID in out assert sample_database.database_id in out @@ -123,8 +121,10 @@ def test_list_backup_operations(capsys, instance_id, sample_database): print(out) -@pytest.mark.dependency(depends=["create_backup"]) -def test_list_backups(capsys, instance_id, sample_database, ): +@pytest.mark.dependency(name="list_backup", depends=["create_backup", "copy_backup"]) +def test_list_backups( + capsys, instance_id, sample_database, +): backup_sample.list_backups( instance_id, sample_database.database_id, BACKUP_ID, ) @@ -140,7 +140,7 @@ def test_update_backup(capsys, instance_id): assert BACKUP_ID in out -@pytest.mark.dependency(depends=["create_backup","copy_backup"]) +@pytest.mark.dependency(depends=["create_backup", "copy_backup", "list_backup"]) def test_delete_backup(capsys, instance_id): backup_sample.delete_backup(instance_id, BACKUP_ID) out, _ = capsys.readouterr() @@ -173,4 +173,3 @@ def test_create_database_with_retention_period(capsys, sample_instance): assert ("retention period " + RETENTION_PERIOD) in out database = sample_instance.database(RETENTION_DATABASE_ID) database.drop() - diff --git a/samples/samples/conftest.py b/samples/samples/conftest.py index b3728a4db4..314c984920 100644 --- a/samples/samples/conftest.py +++ b/samples/samples/conftest.py @@ -93,9 +93,7 @@ def instance_config(spanner_client): @pytest.fixture(scope="module") def multi_region_instance_config(spanner_client): - return "{}/instanceConfigs/{}".format( - spanner_client.project_name, "nam3" - ) + return "{}/instanceConfigs/{}".format(spanner_client.project_name, "nam3") @pytest.fixture(scope="module") @@ -143,7 +141,7 @@ def multi_region_instance( labels={ "cloud_spanner_samples": "true", "sample_name": sample_name, - "created": str(int(time.time())) + "created": str(int(time.time())), }, ) op = retry_429(multi_region_instance.create)() diff --git a/samples/samples/noxfile.py b/samples/samples/noxfile.py index 4c808af73e..85f5836dba 100644 --- a/samples/samples/noxfile.py +++ b/samples/samples/noxfile.py @@ -208,9 +208,7 @@ def _session_tests( if os.path.exists("requirements-test.txt"): if os.path.exists("constraints-test.txt"): - session.install( - "-r", "requirements-test.txt", "-c", "constraints-test.txt" - ) + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") else: session.install("-r", "requirements-test.txt") with open("requirements-test.txt") as rtfile: @@ -223,9 +221,9 @@ def _session_tests( post_install(session) if "pytest-parallel" in packages: - concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) + concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"]) elif "pytest-xdist" in packages: - concurrent_args.extend(['-n', 'auto']) + concurrent_args.extend(["-n", "auto"]) session.run( "pytest", diff --git a/samples/samples/snippets.py b/samples/samples/snippets.py index 5a3ac6df24..87721c021f 100644 --- a/samples/samples/snippets.py +++ b/samples/samples/snippets.py @@ -51,8 +51,8 @@ def create_instance(instance_id): labels={ "cloud_spanner_samples": "true", "sample_name": "snippets-create_instance-explicit", - "created": str(int(time.time())) - } + "created": str(int(time.time())), + }, ) operation = instance.create() @@ -83,8 +83,8 @@ def create_instance_with_processing_units(instance_id, processing_units): labels={ "cloud_spanner_samples": "true", "sample_name": "snippets-create_instance_with_processing_units", - "created": str(int(time.time())) - } + "created": str(int(time.time())), + }, ) operation = instance.create() @@ -92,8 +92,11 @@ def create_instance_with_processing_units(instance_id, processing_units): print("Waiting for operation to complete...") operation.result(OPERATION_TIMEOUT_SECONDS) - print("Created instance {} with {} processing units".format( - instance_id, instance.processing_units)) + print( + "Created instance {} with {} processing units".format( + instance_id, instance.processing_units + ) + ) # [END spanner_create_instance_with_processing_units] @@ -103,10 +106,15 @@ def create_instance_with_processing_units(instance_id, processing_units): def get_instance_config(instance_config): """Gets the leader options for the instance configuration.""" spanner_client = spanner.Client() - config_name = "{}/instanceConfigs/{}".format(spanner_client.project_name, instance_config) + config_name = "{}/instanceConfigs/{}".format( + spanner_client.project_name, instance_config + ) config = spanner_client.instance_admin_api.get_instance_config(name=config_name) - print("Available leader options for instance config {}: {}".format( - instance_config, config.leader_options)) + print( + "Available leader options for instance config {}: {}".format( + instance_config, config.leader_options + ) + ) # [END spanner_get_instance_config] @@ -203,7 +211,7 @@ def create_database_with_encryption_key(instance_id, database_id, kms_key_name): ) PRIMARY KEY (SingerId, AlbumId), INTERLEAVE IN PARENT Singers ON DELETE CASCADE""", ], - encryption_config={'kms_key_name': kms_key_name}, + encryption_config={"kms_key_name": kms_key_name}, ) operation = database.create() @@ -211,17 +219,18 @@ def create_database_with_encryption_key(instance_id, database_id, kms_key_name): print("Waiting for operation to complete...") operation.result(OPERATION_TIMEOUT_SECONDS) - print("Database {} created with encryption key {}".format( - database.name, database.encryption_config.kms_key_name)) + print( + "Database {} created with encryption key {}".format( + database.name, database.encryption_config.kms_key_name + ) + ) # [END spanner_create_database_with_encryption_key] # [START spanner_create_database_with_default_leader] -def create_database_with_default_leader( - instance_id, database_id, default_leader -): +def create_database_with_default_leader(instance_id, database_id, default_leader): """Creates a database with tables with a default leader.""" spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) @@ -254,7 +263,7 @@ def create_database_with_default_leader( print( "Database {} created with default leader {}".format( - database.name, database.default_leader + database.name, database.default_leader ) ) @@ -263,17 +272,19 @@ def create_database_with_default_leader( # [START spanner_update_database_with_default_leader] -def update_database_with_default_leader( - instance_id, database_id, default_leader -): +def update_database_with_default_leader(instance_id, database_id, default_leader): """Updates a database with tables with a default leader.""" spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) database = instance.database(database_id) - operation = database.update_ddl(["ALTER DATABASE {}" - " SET OPTIONS (default_leader = '{}')".format(database_id, default_leader)]) + operation = database.update_ddl( + [ + "ALTER DATABASE {}" + " SET OPTIONS (default_leader = '{}')".format(database_id, default_leader) + ] + ) operation.result(OPERATION_TIMEOUT_SECONDS) database.reload() @@ -316,9 +327,7 @@ def query_information_schema_database_options(instance_id, database_id): "WHERE SCHEMA_NAME = '' AND OPTION_NAME = 'default_leader'" ) for result in results: - print("Database {} has default leader {}".format( - database_id, result[0] - )) + print("Database {} has default leader {}".format(database_id, result[0])) # [END spanner_query_information_schema_database_options] @@ -1307,11 +1316,9 @@ def insert_singers(transaction): database.run_in_transaction(insert_singers) commit_stats = database.logger.last_commit_stats - print( - "{} mutation(s) in transaction.".format( - commit_stats.mutation_count - ) - ) + print("{} mutation(s) in transaction.".format(commit_stats.mutation_count)) + + # [END spanner_get_commit_stats] @@ -2011,7 +2018,7 @@ def query_data_with_query_options(instance_id, database_id): "SELECT VenueId, VenueName, LastUpdateTime FROM Venues", query_options={ "optimizer_version": "1", - "optimizer_statistics_package": "latest" + "optimizer_statistics_package": "latest", }, ) @@ -2028,8 +2035,9 @@ def create_client_with_query_options(instance_id, database_id): spanner_client = spanner.Client( query_options={ "optimizer_version": "1", - "optimizer_statistics_package": "auto_20191128_14_47_22UTC" - }) + "optimizer_statistics_package": "auto_20191128_14_47_22UTC", + } + ) instance = spanner_client.instance(instance_id) database = instance.database(database_id) @@ -2057,7 +2065,7 @@ def update_venues(transaction): # This request tag will only be set on this request. transaction.execute_update( "UPDATE Venues SET Capacity = CAST(Capacity/4 AS INT64) WHERE OutdoorVenue = false", - request_options={"request_tag": "app=concert,env=dev,action=update"} + request_options={"request_tag": "app=concert,env=dev,action=update"}, ) print("Venue capacities updated.") @@ -2070,21 +2078,19 @@ def update_venues(transaction): "venueId": 81, "venueName": "Venue 81", "capacity": 1440, - "outdoorVenue": True + "outdoorVenue": True, }, param_types={ "venueId": param_types.INT64, "venueName": param_types.STRING, "capacity": param_types.INT64, - "outdoorVenue": param_types.BOOL + "outdoorVenue": param_types.BOOL, }, - request_options={"request_tag": "app=concert,env=dev,action=insert"} + request_options={"request_tag": "app=concert,env=dev,action=insert"}, ) print("New venue inserted.") - database.run_in_transaction( - update_venues, transaction_tag="app=concert,env=dev" - ) + database.run_in_transaction(update_venues, transaction_tag="app=concert,env=dev") # [END spanner_set_transaction_tag] @@ -2101,7 +2107,7 @@ def set_request_tag(instance_id, database_id): with database.snapshot() as snapshot: results = snapshot.execute_sql( "SELECT SingerId, AlbumId, AlbumTitle FROM Albums", - request_options={"request_tag": "app=concert,env=dev,action=select"} + request_options={"request_tag": "app=concert,env=dev,action=select"}, ) for row in results: diff --git a/samples/samples/snippets_test.py b/samples/samples/snippets_test.py index d81032fa20..a5fa6a5caf 100644 --- a/samples/samples/snippets_test.py +++ b/samples/samples/snippets_test.py @@ -124,8 +124,12 @@ def test_create_instance_with_processing_units(capsys, lci_instance_id): retry_429(instance.delete)() -def test_create_database_with_encryption_config(capsys, instance_id, cmek_database_id, kms_key_name): - snippets.create_database_with_encryption_key(instance_id, cmek_database_id, kms_key_name) +def test_create_database_with_encryption_config( + capsys, instance_id, cmek_database_id, kms_key_name +): + snippets.create_database_with_encryption_key( + instance_id, cmek_database_id, kms_key_name + ) out, _ = capsys.readouterr() assert cmek_database_id in out assert kms_key_name in out @@ -150,7 +154,13 @@ def test_list_databases(capsys, instance_id): assert "has default leader" in out -def test_create_database_with_default_leader(capsys, multi_region_instance, multi_region_instance_id, default_leader_database_id, default_leader): +def test_create_database_with_default_leader( + capsys, + multi_region_instance, + multi_region_instance_id, + default_leader_database_id, + default_leader, +): retry_429 = RetryErrors(exceptions.ResourceExhausted, delay=15) retry_429(snippets.create_database_with_default_leader)( multi_region_instance_id, default_leader_database_id, default_leader @@ -160,7 +170,13 @@ def test_create_database_with_default_leader(capsys, multi_region_instance, mult assert default_leader in out -def test_update_database_with_default_leader(capsys, multi_region_instance, multi_region_instance_id, default_leader_database_id, default_leader): +def test_update_database_with_default_leader( + capsys, + multi_region_instance, + multi_region_instance_id, + default_leader_database_id, + default_leader, +): retry_429 = RetryErrors(exceptions.ResourceExhausted, delay=15) retry_429(snippets.update_database_with_default_leader)( multi_region_instance_id, default_leader_database_id, default_leader @@ -176,7 +192,13 @@ def test_get_database_ddl(capsys, instance_id, sample_database): assert sample_database.database_id in out -def test_query_information_schema_database_options(capsys, multi_region_instance, multi_region_instance_id, default_leader_database_id, default_leader): +def test_query_information_schema_database_options( + capsys, + multi_region_instance, + multi_region_instance_id, + default_leader_database_id, + default_leader, +): snippets.query_information_schema_database_options( multi_region_instance_id, default_leader_database_id ) @@ -587,7 +609,9 @@ def test_query_data_with_json_parameter(capsys, instance_id, sample_database): @pytest.mark.dependency(depends=["insert_datatypes_data"]) def test_query_data_with_timestamp_parameter(capsys, instance_id, sample_database): - snippets.query_data_with_timestamp_parameter(instance_id, sample_database.database_id) + snippets.query_data_with_timestamp_parameter( + instance_id, sample_database.database_id + ) out, _ = capsys.readouterr() assert "VenueId: 4, VenueName: Venue 4, LastUpdateTime:" in out assert "VenueId: 19, VenueName: Venue 19, LastUpdateTime:" in out