From f977695fd6e64fdc781bd0dd15d509448a86e8a1 Mon Sep 17 00:00:00 2001 From: trevor-pope Date: Sun, 12 Jun 2022 22:56:59 -0400 Subject: [PATCH 01/15] Added parenthesis to enclose DEFAULT values of CREATE TABLE statements and changed order of NOT NULL and DEFAULT statements. --- google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py b/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py index 82eb14bd..2a2a0363 100644 --- a/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py +++ b/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py @@ -342,13 +342,13 @@ def get_column_specification(self, column, **kwargs): + " " + self.dialect.type_compiler.process(column.type, type_expression=column) ) - default = self.get_column_default_string(column) - if default is not None: - colspec += " DEFAULT " + default - if not column.nullable: colspec += " NOT NULL" + default = self.get_column_default_string(column) + if default is not None: + colspec += " DEFAULT (" + default + ")" + if column.computed is not None: colspec += " " + self.process(column.computed) From ac87cdb4a787108f5b32f9df3252d8770f363be8 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:42:39 -0700 Subject: [PATCH 02/15] Update setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9f89d405..e8c29328 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import os import setuptools +from pkgutil import extend_path # Package metadata. @@ -49,9 +50,9 @@ ] # Determine which namespaces are needed. -namespaces = ["google"] +__path__ = extend_path(__path__, "google") if "google.cloud" in packages: - namespaces.append("google.cloud") + __path__ = extend_path(__path__, "google.cloud") setuptools.setup( author="Google LLC", @@ -66,7 +67,6 @@ install_requires=dependencies, extras_require=extras, name=name, - namespace_packages=namespaces, packages=packages, url="https://github.com/cloudspannerecosystem/python-spanner-sqlalchemy", version=version, From 00a8638713ac54aad59ac850b75c346e6c4c2b30 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:45:06 -0700 Subject: [PATCH 03/15] Update setup.py --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e8c29328..b38d65d6 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ # limitations under the License. import os +import pkg_resources import setuptools -from pkgutil import extend_path # Package metadata. @@ -50,9 +50,7 @@ ] # Determine which namespaces are needed. -__path__ = extend_path(__path__, "google") -if "google.cloud" in packages: - __path__ = extend_path(__path__, "google.cloud") +pkg_resources.declare_namespace("google") setuptools.setup( author="Google LLC", From 94dae0d53cde4038380b3cdb501777b3f0fe4f76 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:47:34 -0700 Subject: [PATCH 04/15] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b38d65d6..bd500b21 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ # Determine which namespaces are needed. pkg_resources.declare_namespace("google") +pkg_resources.declare_namespace("google.cloud") setuptools.setup( author="Google LLC", From dd2be48617fcd714e280ca27dc553b25f2a843dd Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:48:56 -0700 Subject: [PATCH 05/15] Update setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bd500b21..414d90d9 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,7 @@ ] # Determine which namespaces are needed. -pkg_resources.declare_namespace("google") -pkg_resources.declare_namespace("google.cloud") +pkg_resources.declare_namespace("google.cloud.sqlalchemy_spanner") setuptools.setup( author="Google LLC", From 8de6214b25c0263bee62b2d9d17a4e4f48848bff Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:53:20 -0700 Subject: [PATCH 06/15] Update setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 414d90d9..ec6f1e1e 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import pkg_resources import setuptools @@ -50,7 +49,7 @@ ] # Determine which namespaces are needed. -pkg_resources.declare_namespace("google.cloud.sqlalchemy_spanner") +__import__("pkg_resources").declare_namespace(__name__) setuptools.setup( author="Google LLC", From fbc90adda2b1d772ca9a73d72a64cd6c3d6ac078 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 13 Jul 2022 23:58:58 -0700 Subject: [PATCH 07/15] Update setup.py --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ec6f1e1e..6288e6f5 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,9 @@ ] # Determine which namespaces are needed. -__import__("pkg_resources").declare_namespace(__name__) +namespaces = ["google"] +if "google.cloud" in packages: + namespaces.append("google.cloud" setuptools.setup( author="Google LLC", @@ -64,6 +66,7 @@ install_requires=dependencies, extras_require=extras, name=name, + namespace_packages=namespaces, packages=packages, url="https://github.com/cloudspannerecosystem/python-spanner-sqlalchemy", version=version, From 4feb517d37ea5ef1c4e517fd7f034b04dce6b75d Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 14 Jul 2022 00:00:22 -0700 Subject: [PATCH 08/15] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6288e6f5..9f89d405 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ # Determine which namespaces are needed. namespaces = ["google"] if "google.cloud" in packages: - namespaces.append("google.cloud" + namespaces.append("google.cloud") setuptools.setup( author="Google LLC", From 48c8251e466bbfb10a0a184b3f8499e3734cce54 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:00:06 -0700 Subject: [PATCH 09/15] Update setup.py --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 9f89d405..0d60c839 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,10 @@ exec(f.read(), PACKAGE_INFO) version = PACKAGE_INFO["__version__"] +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + # Only include packages under the 'google' namespace. Do not include tests, # benchmarks, etc. packages = [ @@ -58,6 +62,7 @@ author_email="cloud-spanner-developers@googlegroups.com", classifiers=["Intended Audience :: Developers"], description=description, + long_description=readme, entry_points={ "sqlalchemy.dialects": [ "spanner.spanner = google.cloud.sqlalchemy_spanner:SpannerDialect" From 1217cef0fdfba18b992cb4494b73a71afceaf4d5 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:01:47 -0700 Subject: [PATCH 10/15] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0d60c839..794c2e9b 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ exec(f.read(), PACKAGE_INFO) version = PACKAGE_INFO["__version__"] +package_root = os.path.abspath(os.path.dirname(__file__)) readme_filename = os.path.join(package_root, "README.rst") with io.open(readme_filename, encoding="utf-8") as readme_file: readme = readme_file.read() From 3da722162a246eb64b7753d3b5903f4e658b8c74 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:03:24 -0700 Subject: [PATCH 11/15] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 794c2e9b..72fa5653 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import os import setuptools From 5954b25d396d81b5969cb28164d78ff802f98468 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:05:19 -0700 Subject: [PATCH 12/15] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 72fa5653..90e4547c 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ version = PACKAGE_INFO["__version__"] package_root = os.path.abspath(os.path.dirname(__file__)) -readme_filename = os.path.join(package_root, "README.rst") +readme_filename = os.path.join(package_root, "README.me") with io.open(readme_filename, encoding="utf-8") as readme_file: readme = readme_file.read() From 42e25e5c09c31aed82c24470471c21893ae5c22d Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:05:47 -0700 Subject: [PATCH 13/15] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90e4547c..6d5141dc 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ version = PACKAGE_INFO["__version__"] package_root = os.path.abspath(os.path.dirname(__file__)) -readme_filename = os.path.join(package_root, "README.me") +readme_filename = os.path.join(package_root, "README.md") with io.open(readme_filename, encoding="utf-8") as readme_file: readme = readme_file.read() From 0e7c17eba1a7fac3f8f08e5000cdd580897ae992 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:11:52 -0700 Subject: [PATCH 14/15] Update and rename README.md to README.rst --- README.md | 340 ------------------------------------ README.rst | 494 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+), 340 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index e348f597..00000000 --- a/README.md +++ /dev/null @@ -1,340 +0,0 @@ -# Spanner dialect for SQLAlchemy - -Spanner dialect for SQLAlchemy represents an interface API designed to make it possible to control Cloud Spanner databases with SQLAlchemy API. The dialect is built on top of [the Spanner DB API](https://github.com/googleapis/python-spanner/tree/master/google/cloud/spanner_dbapi), which is designed in accordance with [PEP-249](https://www.python.org/dev/peps/pep-0249/). - -Known limitations are listed [here](#features-and-limitations). All supported features have been tested and verified to work with the test configurations. There may be configurations and/or data model variations that have not yet been covered by the tests and that show unexpected behavior. Please report any problems that you might encounter by [creating a new issue](https://github.com/googleapis/python-spanner-sqlalchemy/issues/new). - -- [Cloud Spanner product documentation](https://cloud.google.com/spanner/docs) -- [SQLAlchemy product documentation](https://www.sqlalchemy.org/) - -## Quick Start - -In order to use this package, you first need to go through the following steps: - -1. [Select or create a Cloud Platform project.](https://console.cloud.google.com/project) -2. [Enable billing for your project.](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project) -3. [Enable the Google Cloud Spanner API.](https://cloud.google.com/spanner) -4. [Setup Authentication.](https://googleapis.dev/python/google-api-core/latest/auth.html) - -## Installation - -To install an in-development version of the package, clone its Git-repository: -``` -git clone https://github.com/googleapis/python-spanner-sqlalchemy.git -``` -Next install the package from the package `setup.py` file: -``` -python setup.py install -``` -During setup the dialect will be registered with entry points. - -## A Minimal App - -### Database URL -In order to connect to a database one have to use its URL on connection creation step. SQLAlchemy 1.3 and 1.4 versions have a bit of difference on this step in a dialect prefix part: -```python -# for SQLAlchemy 1.3: -spanner:///projects/project-id/instances/instance-id/databases/database-id - -# for SQLAlchemy 1.4: -spanner+spanner:///projects/project-id/instances/instance-id/databases/database-id -``` - -### Create a table -```python -from sqlalchemy import ( - Column, - Integer, - MetaData, - String, - Table, - create_engine, -) - -engine = create_engine( - "spanner:///projects/project-id/instances/instance-id/databases/database-id" -) -metadata = MetaData(bind=engine) - -user = Table( - "users", - metadata, - Column("user_id", Integer, primary_key=True), - Column("user_name", String(16), nullable=False), -) - -metadata.create_all(engine) -``` - -### Insert a row -```python -import uuid - -from sqlalchemy import ( - MetaData, - Table, - create_engine, -) - -engine = create_engine( - "spanner:///projects/project-id/instances/instance-id/databases/database-id" -) -user = Table("users", MetaData(bind=engine), autoload=True) -user_id = uuid.uuid4().hex[:6].lower() - -with engine.begin() as connection: - connection.execute(user.insert(), {"user_id": user_id, "user_name": "Full Name"}) -``` - -### Read -```python -from sqlalchemy import MetaData, Table, create_engine, select - -engine = create_engine( - "spanner:///projects/project-id/instances/instance-id/databases/database-id" -) -table = Table("users", MetaData(bind=engine), autoload=True) - -with engine.begin() as connection: - for row in connection.execute(select(["*"], from_obj=table)).fetchall(): - print(row) -``` - -## Migration - -SQLAlchemy uses [Alembic](https://alembic.sqlalchemy.org/en/latest/#) tool to organize database migrations. - -Spanner dialect doesn't provide a default migration environment, it's up to user to write it. One thing to be noted here - one should explicitly set `alembic_version` table not to use migration revision id as a primary key: -```python -with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - version_table_pk=False, # don't use primary key in the versions table - ) -``` -As Spanner restricts changing a primary key value, not setting the flag to `False` can cause migration problems. - -**Warning!** -A migration script can produce a lot of DDL statements. If each of the statements are executed separately, performance issues can occur. To avoid these, it's highly recommended to use the [Alembic batch context](https://alembic.sqlalchemy.org/en/latest/batch.html) feature to pack DDL statements into groups of statements. - - -## Features and limitations - -### Interleaved tables -Cloud Spanner dialect includes two dialect-specific arguments for `Table` constructor, which help to define interleave relations: -`spanner_interleave_in` - a parent table name -`spanner_inverleave_on_delete_cascade` - a flag specifying if `ON DELETE CASCADE` statement must be used for the interleave relation -An example of interleave relations definition: -```python -team = Table( - "team", - metadata, - Column("team_id", Integer, primary_key=True), - Column("team_name", String(16), nullable=False), -) -team.create(engine) - -client = Table( - "client", - metadata, - Column("team_id", Integer, primary_key=True), - Column("client_id", Integer, primary_key=True), - Column("client_name", String(16), nullable=False), - spanner_interleave_in="team", - spanner_interleave_on_delete_cascade=True, -) -client.add_is_dependent_on(team) - -client.create(engine) -``` -**Note**: Interleaved tables have a dependency between them, so the parent table must be created before the child table. When creating tables with this feature, make sure to call `add_is_dependent_on()` on the child table to request SQLAlchemy to create the parent table before the child table. - -### Unique constraints -Cloud Spanner doesn't support direct UNIQUE constraints creation. In order to achieve column values uniqueness UNIQUE indexes should be used. - -Instead of direct UNIQUE constraint creation: -```python -Table( - 'table', - metadata, - Column('col1', Integer), - UniqueConstraint('col1', name='uix_1') -) -``` -Create a UNIQUE index: -```python -Table( - 'table', - metadata, - Column('col1', Integer), - Index("uix_1", "col1", unique=True), -) -``` -### Autocommit mode -Spanner dialect supports both `SERIALIZABLE` and `AUTOCOMMIT` isolation levels. `SERIALIZABLE` is the default one, where transactions need to be committed manually. `AUTOCOMMIT` mode corresponds to automatically committing of a query right in its execution time. - -Isolation level change example: -```python -from sqlalchemy import create_engine - -eng = create_engine("spanner:///projects/project-id/instances/instance-id/databases/database-id") -autocommit_engine = eng.execution_options(isolation_level="AUTOCOMMIT") -``` - -### Autoincremented IDs -Cloud Spanner doesn't support autoincremented IDs mechanism due to performance reasons ([see for more details](https://cloud.google.com/spanner/docs/schema-design#primary-key-prevent-hotspots)). We recommend that you use the Python [uuid](https://docs.python.org/3/library/uuid.html) module to generate primary key fields to avoid creating monotonically increasing keys. - -Though it's not encouraged to do so, in case you *need* the feature, you can simulate it manually as follows: -```python -with engine.begin() as connection: - top_id = connection.execute( - select([user.c.user_id]).order_by(user.c.user_id.desc()).limit(1) - ).fetchone() - next_id = top_id[0] + 1 if top_id else 1 - - connection.execute(user.insert(), {"user_id": next_id}) -``` - -### Query hints -Spanner dialect supports [query hints](https://cloud.google.com/spanner/docs/query-syntax#table_hints), which give the ability to set additional query execution parameters. Usage example: -```python -session = Session(engine) - -Base = declarative_base() - -class User(Base): - """Data model.""" - - __tablename__ = "users" - id = Column(Integer, primary_key=True) - name = Column(String(50)) - - -query = session.query(User) -query = query.with_hint( - selectable=User, text="@{FORCE_INDEX=index_name}" -) -query = query.filter(User.name.in_(["val1", "val2"])) -query.statement.compile(session.bind) -``` - -### ReadOnly transactions -By default, transactions produced by a Spanner connection are in ReadWrite mode. However, some applications require an ability to grant ReadOnly access to users/methods; for these cases Spanner dialect supports the `read_only` execution option, which switches a connection into ReadOnly mode: -```python -with engine.connect().execution_options(read_only=True) as connection: - connection.execute(select(["*"], from_obj=table)).fetchall() -``` -Note that execution options are applied lazily - on the `execute()` method call, right before it. - -ReadOnly/ReadWrite mode of a connection can't be changed while a transaction is in progress - first you must commit or rollback it. - -### Stale reads -To use the Spanner [Stale Reads](https://cloud.google.com/spanner/docs/reads#perform-stale-read) with SQLAlchemy you can tweak the connection execution options with a wanted staleness value. For example: -```python -# maximum staleness -with engine.connect().execution_options( - read_only=True, - staleness={"max_staleness": datetime.timedelta(seconds=5)} -) as connection: - connection.execute(select(["*"], from_obj=table)).fetchall() -``` - -```python -# exact staleness -with engine.connect().execution_options( - read_only=True, - staleness={"exact_staleness": datetime.timedelta(seconds=5)} -) as connection: - connection.execute(select(["*"], from_obj=table)).fetchall() -``` - -```python -# min read timestamp -with engine.connect().execution_options( - read_only=True, - staleness={"min_read_timestamp": datetime.datetime(2021, 11, 17, 12, 55, 30)} -) as connection: - connection.execute(select(["*"], from_obj=table)).fetchall() -``` - -```python -# read timestamp -with engine.connect().execution_options( - read_only=True, - staleness={"read_timestamp": datetime.datetime(2021, 11, 17, 12, 55, 30)} -) as connection: - connection.execute(select(["*"], from_obj=table)).fetchall() -``` -Note that the set option will be dropped when the connection is returned back to the pool. - -### DDL and transactions -DDL statements are executed outside the regular transactions mechanism, which means DDL statements will not be rolled back on normal transaction rollback. - -### Dropping a table -Cloud Spanner, by default, doesn't drop tables, which have secondary indexes and/or foreign key constraints. In Spanner dialect for SQLAlchemy, however, this restriction is omitted - if a table you are trying to delete has indexes/foreign keys, they will be dropped automatically right before dropping the table. - -### Data types -Data types table mapping SQLAlchemy types to Cloud Spanner types: - -| SQLAlchemy | Spanner | -| ------------- | ------------- | -| INTEGER | INT64 | -| BIGINT | INT64 | -| DECIMAL | NUMERIC | -| FLOAT | FLOAT64 | -| TEXT | STRING | -| ARRAY | ARRAY | -| BINARY | BYTES | -| VARCHAR | STRING | -| CHAR | STRING | -| BOOLEAN | BOOL | -| DATETIME | TIMESTAMP | -| NUMERIC | NUMERIC | - - -### Other limitations -- WITH RECURSIVE statement is not supported. -- Named schemas are not supported. -- Temporary tables are not supported. -- Numeric type dimensions (scale and precision) are constant. See the [docs](https://cloud.google.com/spanner/docs/data-types#numeric_types). - -## Best practices -When a SQLAlchemy function is called, a new connection to a database is established and a Spanner session object is fetched. In case of connectionless execution these fetches are done for every `execute()` call, which can cause a significant latency. To avoid initiating a Spanner session on every `execute()` call it's recommended to write code in connection-bounded fashion. Once a `Connection()` object is explicitly initiated, it fetches a Spanner session object and uses it for all the following calls made on this `Connection()` object. - -Non-optimal connectionless use: -```python -# execute() is called on object, which is not a Connection() object -insert(user).values(user_id=1, user_name="Full Name").execute() -``` -Optimal connection-bounded use: -```python -with engine.begin() as connection: - # execute() is called on a Connection() object - connection.execute(user.insert(), {"user_id": 1, "user_name": "Full Name"}) -``` -Connectionless way of use is also deprecated since SQLAlchemy 2.0 and soon will be removed (see in [SQLAlchemy docs](https://docs.sqlalchemy.org/en/14/core/connections.html#connectionless-execution-implicit-execution)). - -## Running tests - -Spanner dialect includes a compliance, migration and unit test suite. To run the tests the `nox` package commands can be used: -``` -# Run the whole suite -$ nox - -# Run a particular test session -$ nox -s migration_test -``` -### Running tests on Spanner emulator -The dialect test suite can be runned on [Spanner emulator](https://cloud.google.com/spanner/docs/emulator). Several tests, relating to `NULL` values of data types, are skipped when executed on emulator. - -## Contributing - -Contributions to this library are welcome and encouraged. Please report issues, file feature requests, and send pull requests. See [CONTRIBUTING](https://github.com/googleapis/python-spanner-sqlalchemy/blob/main/contributing.md) for more information on how to get -started. - -**Note that this project is not officially supported by Google as part of the Cloud Spanner product.** - -Please note that this project is released with a Contributor Code of Conduct. -By participating in this project you agree to abide by its terms. See the [Code -of Conduct](https://github.com/googleapis/python-spanner-sqlalchemy/blob/main/code-of-conduct.md) for more information. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..b4550cd2 --- /dev/null +++ b/README.rst @@ -0,0 +1,494 @@ +Spanner dialect for SQLAlchemy +============================== + +Spanner dialect for SQLAlchemy represents an interface API designed to +make it possible to control Cloud Spanner databases with SQLAlchemy API. +The dialect is built on top of `the Spanner DB +API `__, +which is designed in accordance with +`PEP-249 `__. + +Known limitations are listed `here <#features-and-limitations>`__. All +supported features have been tested and verified to work with the test +configurations. There may be configurations and/or data model variations +that have not yet been covered by the tests and that show unexpected +behavior. Please report any problems that you might encounter by +`creating a new +issue `__. + +- `Cloud Spanner product + documentation `__ +- `SQLAlchemy product documentation `__ + +Quick Start +----------- + +In order to use this package, you first need to go through the following +steps: + +1. `Select or create a Cloud Platform + project. `__ +2. `Enable billing for your + project. `__ +3. `Enable the Google Cloud Spanner + API. `__ +4. `Setup + Authentication. `__ + +Installation +------------ + +To install an in-development version of the package, clone its +Git-repository: + +:: + + git clone https://github.com/googleapis/python-spanner-sqlalchemy.git + +Next install the package from the package ``setup.py`` file: + +:: + + python setup.py install + +During setup the dialect will be registered with entry points. + +A Minimal App +------------- + +Database URL +~~~~~~~~~~~~ + +In order to connect to a database one have to use its URL on connection +creation step. SQLAlchemy 1.3 and 1.4 versions have a bit of difference +on this step in a dialect prefix part: + +.. code:: python + + # for SQLAlchemy 1.3: + spanner:///projects/project-id/instances/instance-id/databases/database-id + + # for SQLAlchemy 1.4: + spanner+spanner:///projects/project-id/instances/instance-id/databases/database-id + +Create a table +~~~~~~~~~~~~~~ + +.. code:: python + + from sqlalchemy import ( + Column, + Integer, + MetaData, + String, + Table, + create_engine, + ) + + engine = create_engine( + "spanner:///projects/project-id/instances/instance-id/databases/database-id" + ) + metadata = MetaData(bind=engine) + + user = Table( + "users", + metadata, + Column("user_id", Integer, primary_key=True), + Column("user_name", String(16), nullable=False), + ) + + metadata.create_all(engine) + +Insert a row +~~~~~~~~~~~~ + +.. code:: python + + import uuid + + from sqlalchemy import ( + MetaData, + Table, + create_engine, + ) + + engine = create_engine( + "spanner:///projects/project-id/instances/instance-id/databases/database-id" + ) + user = Table("users", MetaData(bind=engine), autoload=True) + user_id = uuid.uuid4().hex[:6].lower() + + with engine.begin() as connection: + connection.execute(user.insert(), {"user_id": user_id, "user_name": "Full Name"}) + +Read +~~~~ + +.. code:: python + + from sqlalchemy import MetaData, Table, create_engine, select + + engine = create_engine( + "spanner:///projects/project-id/instances/instance-id/databases/database-id" + ) + table = Table("users", MetaData(bind=engine), autoload=True) + + with engine.begin() as connection: + for row in connection.execute(select(["*"], from_obj=table)).fetchall(): + print(row) + +Migration +--------- + +SQLAlchemy uses `Alembic `__ +tool to organize database migrations. + +Spanner dialect doesn't provide a default migration environment, it's up +to user to write it. One thing to be noted here - one should explicitly +set ``alembic_version`` table not to use migration revision id as a +primary key: + +.. code:: python + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table_pk=False, # don't use primary key in the versions table + ) + +As Spanner restricts changing a primary key value, not setting the flag +to ``False`` can cause migration problems. + +| **Warning!** +| A migration script can produce a lot of DDL statements. If each of the + statements are executed separately, performance issues can occur. To + avoid these, it's highly recommended to use the `Alembic batch + context `__ + feature to pack DDL statements into groups of statements. + +Features and limitations +------------------------ + +Interleaved tables +~~~~~~~~~~~~~~~~~~ + +| Cloud Spanner dialect includes two dialect-specific arguments for + ``Table`` constructor, which help to define interleave relations: + ``spanner_interleave_in`` - a parent table name + ``spanner_inverleave_on_delete_cascade`` - a flag specifying if + ``ON DELETE CASCADE`` statement must be used for the interleave + relation +| An example of interleave relations definition: + +.. code:: python + + team = Table( + "team", + metadata, + Column("team_id", Integer, primary_key=True), + Column("team_name", String(16), nullable=False), + ) + team.create(engine) + + client = Table( + "client", + metadata, + Column("team_id", Integer, primary_key=True), + Column("client_id", Integer, primary_key=True), + Column("client_name", String(16), nullable=False), + spanner_interleave_in="team", + spanner_interleave_on_delete_cascade=True, + ) + client.add_is_dependent_on(team) + + client.create(engine) + +**Note**: Interleaved tables have a dependency between them, so the +parent table must be created before the child table. When creating +tables with this feature, make sure to call ``add_is_dependent_on()`` on +the child table to request SQLAlchemy to create the parent table before +the child table. + +Unique constraints +~~~~~~~~~~~~~~~~~~ + +Cloud Spanner doesn't support direct UNIQUE constraints creation. In +order to achieve column values uniqueness UNIQUE indexes should be used. + +Instead of direct UNIQUE constraint creation: + +.. code:: python + + Table( + 'table', + metadata, + Column('col1', Integer), + UniqueConstraint('col1', name='uix_1') + ) + +Create a UNIQUE index: + +.. code:: python + + Table( + 'table', + metadata, + Column('col1', Integer), + Index("uix_1", "col1", unique=True), + ) + +Autocommit mode +~~~~~~~~~~~~~~~ + +Spanner dialect supports both ``SERIALIZABLE`` and ``AUTOCOMMIT`` +isolation levels. ``SERIALIZABLE`` is the default one, where +transactions need to be committed manually. ``AUTOCOMMIT`` mode +corresponds to automatically committing of a query right in its +execution time. + +Isolation level change example: + +.. code:: python + + from sqlalchemy import create_engine + + eng = create_engine("spanner:///projects/project-id/instances/instance-id/databases/database-id") + autocommit_engine = eng.execution_options(isolation_level="AUTOCOMMIT") + +Autoincremented IDs +~~~~~~~~~~~~~~~~~~~ + +Cloud Spanner doesn't support autoincremented IDs mechanism due to +performance reasons (`see for more +details `__). +We recommend that you use the Python +`uuid `__ module to +generate primary key fields to avoid creating monotonically increasing +keys. + +Though it's not encouraged to do so, in case you *need* the feature, you +can simulate it manually as follows: + +.. code:: python + + with engine.begin() as connection: + top_id = connection.execute( + select([user.c.user_id]).order_by(user.c.user_id.desc()).limit(1) + ).fetchone() + next_id = top_id[0] + 1 if top_id else 1 + + connection.execute(user.insert(), {"user_id": next_id}) + +Query hints +~~~~~~~~~~~ + +Spanner dialect supports `query +hints `__, +which give the ability to set additional query execution parameters. +Usage example: + +.. code:: python + + session = Session(engine) + + Base = declarative_base() + + class User(Base): + """Data model.""" + + __tablename__ = "users" + id = Column(Integer, primary_key=True) + name = Column(String(50)) + + + query = session.query(User) + query = query.with_hint( + selectable=User, text="@{FORCE_INDEX=index_name}" + ) + query = query.filter(User.name.in_(["val1", "val2"])) + query.statement.compile(session.bind) + +ReadOnly transactions +~~~~~~~~~~~~~~~~~~~~~ + +By default, transactions produced by a Spanner connection are in +ReadWrite mode. However, some applications require an ability to grant +ReadOnly access to users/methods; for these cases Spanner dialect +supports the ``read_only`` execution option, which switches a connection +into ReadOnly mode: + +.. code:: python + + with engine.connect().execution_options(read_only=True) as connection: + connection.execute(select(["*"], from_obj=table)).fetchall() + +Note that execution options are applied lazily - on the ``execute()`` +method call, right before it. + +ReadOnly/ReadWrite mode of a connection can't be changed while a +transaction is in progress - first you must commit or rollback it. + +Stale reads +~~~~~~~~~~~ + +To use the Spanner `Stale +Reads `__ +with SQLAlchemy you can tweak the connection execution options with a +wanted staleness value. For example: + +.. code:: python + + # maximum staleness + with engine.connect().execution_options( + read_only=True, + staleness={"max_staleness": datetime.timedelta(seconds=5)} + ) as connection: + connection.execute(select(["*"], from_obj=table)).fetchall() + +.. code:: python + + # exact staleness + with engine.connect().execution_options( + read_only=True, + staleness={"exact_staleness": datetime.timedelta(seconds=5)} + ) as connection: + connection.execute(select(["*"], from_obj=table)).fetchall() + +.. code:: python + + # min read timestamp + with engine.connect().execution_options( + read_only=True, + staleness={"min_read_timestamp": datetime.datetime(2021, 11, 17, 12, 55, 30)} + ) as connection: + connection.execute(select(["*"], from_obj=table)).fetchall() + +.. code:: python + + # read timestamp + with engine.connect().execution_options( + read_only=True, + staleness={"read_timestamp": datetime.datetime(2021, 11, 17, 12, 55, 30)} + ) as connection: + connection.execute(select(["*"], from_obj=table)).fetchall() + +Note that the set option will be dropped when the connection is returned +back to the pool. + +DDL and transactions +~~~~~~~~~~~~~~~~~~~~ + +DDL statements are executed outside the regular transactions mechanism, +which means DDL statements will not be rolled back on normal transaction +rollback. + +Dropping a table +~~~~~~~~~~~~~~~~ + +Cloud Spanner, by default, doesn't drop tables, which have secondary +indexes and/or foreign key constraints. In Spanner dialect for +SQLAlchemy, however, this restriction is omitted - if a table you are +trying to delete has indexes/foreign keys, they will be dropped +automatically right before dropping the table. + +Data types +~~~~~~~~~~ + +Data types table mapping SQLAlchemy types to Cloud Spanner types: + +========== ========= +SQLAlchemy Spanner +========== ========= +INTEGER INT64 +BIGINT INT64 +DECIMAL NUMERIC +FLOAT FLOAT64 +TEXT STRING +ARRAY ARRAY +BINARY BYTES +VARCHAR STRING +CHAR STRING +BOOLEAN BOOL +DATETIME TIMESTAMP +NUMERIC NUMERIC +========== ========= + +Other limitations +~~~~~~~~~~~~~~~~~ + +- WITH RECURSIVE statement is not supported. +- Named schemas are not supported. +- Temporary tables are not supported. +- Numeric type dimensions (scale and precision) are constant. See the + `docs `__. + +Best practices +-------------- + +When a SQLAlchemy function is called, a new connection to a database is +established and a Spanner session object is fetched. In case of +connectionless execution these fetches are done for every ``execute()`` +call, which can cause a significant latency. To avoid initiating a +Spanner session on every ``execute()`` call it's recommended to write +code in connection-bounded fashion. Once a ``Connection()`` object is +explicitly initiated, it fetches a Spanner session object and uses it +for all the following calls made on this ``Connection()`` object. + +Non-optimal connectionless use: + +.. code:: python + + # execute() is called on object, which is not a Connection() object + insert(user).values(user_id=1, user_name="Full Name").execute() + +Optimal connection-bounded use: + +.. code:: python + + with engine.begin() as connection: + # execute() is called on a Connection() object + connection.execute(user.insert(), {"user_id": 1, "user_name": "Full Name"}) + +Connectionless way of use is also deprecated since SQLAlchemy 2.0 and +soon will be removed (see in `SQLAlchemy +docs `__). + +Running tests +------------- + +Spanner dialect includes a compliance, migration and unit test suite. To +run the tests the ``nox`` package commands can be used: + +:: + + # Run the whole suite + $ nox + + # Run a particular test session + $ nox -s migration_test + +Running tests on Spanner emulator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The dialect test suite can be runned on `Spanner +emulator `__. Several +tests, relating to ``NULL`` values of data types, are skipped when +executed on emulator. + +Contributing +------------ + +Contributions to this library are welcome and encouraged. Please report +issues, file feature requests, and send pull requests. See +`CONTRIBUTING `__ +for more information on how to get started. + +**Note that this project is not officially supported by Google as part +of the Cloud Spanner product.** + +Please note that this project is released with a Contributor Code of +Conduct. By participating in this project you agree to abide by its +terms. See the `Code of +Conduct `__ +for more information. From 869e5a6b9f5ef6678c3c9d55d3dad28904bc13d9 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 21 Jul 2022 01:13:07 -0700 Subject: [PATCH 15/15] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d5141dc..72fa5653 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ version = PACKAGE_INFO["__version__"] package_root = os.path.abspath(os.path.dirname(__file__)) -readme_filename = os.path.join(package_root, "README.md") +readme_filename = os.path.join(package_root, "README.rst") with io.open(readme_filename, encoding="utf-8") as readme_file: readme = readme_file.read()