From bc5ddc3fb1eb7eff9a266fe3d1c3c8a4a6fd3763 Mon Sep 17 00:00:00 2001
From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com>
Date: Mon, 4 Oct 2021 21:39:48 +0530
Subject: [PATCH 1/4] fix: add support for json data type (#593)

* fix: add support for json data type

* fix: skip json test for emulator

* refactor: move JsonObject data type to spanner_v1/types/datatypes.py

* refactor: remove duplicate import

* refactor: remove extra connection creation in test

* refactor: move data_types.py file to google/cloud/spanner_v1/

* fix: increased db version time to current time, to give db backup more time

* fix: undo database_version_time method definition.
---
 google/cloud/spanner_dbapi/parse_utils.py    |  2 +
 google/cloud/spanner_v1/__init__.py          |  3 ++
 google/cloud/spanner_v1/_helpers.py          |  7 +++-
 google/cloud/spanner_v1/data_types.py        | 25 ++++++++++++
 tests/system/test_backup_api.py              |  5 +++
 tests/system/test_dbapi.py                   | 41 +++++++++++++++++++-
 tests/unit/spanner_dbapi/test_parse_utils.py | 11 ++++--
 7 files changed, 89 insertions(+), 5 deletions(-)
 create mode 100644 google/cloud/spanner_v1/data_types.py

diff --git a/google/cloud/spanner_dbapi/parse_utils.py b/google/cloud/spanner_dbapi/parse_utils.py
index d967330cea..4f55a7b2c4 100644
--- a/google/cloud/spanner_dbapi/parse_utils.py
+++ b/google/cloud/spanner_dbapi/parse_utils.py
@@ -21,6 +21,7 @@
 
 import sqlparse
 from google.cloud import spanner_v1 as spanner
+from google.cloud.spanner_v1 import JsonObject
 
 from .exceptions import Error, ProgrammingError
 from .parser import parse_values
@@ -38,6 +39,7 @@
     DateStr: spanner.param_types.DATE,
     TimestampStr: spanner.param_types.TIMESTAMP,
     decimal.Decimal: spanner.param_types.NUMERIC,
+    JsonObject: spanner.param_types.JSON,
 }
 
 SPANNER_RESERVED_KEYWORDS = {
diff --git a/google/cloud/spanner_v1/__init__.py b/google/cloud/spanner_v1/__init__.py
index 4ece165503..4aa08d2c29 100644
--- a/google/cloud/spanner_v1/__init__.py
+++ b/google/cloud/spanner_v1/__init__.py
@@ -58,6 +58,7 @@
 from .types.type import StructType
 from .types.type import Type
 from .types.type import TypeCode
+from .data_types import JsonObject
 
 from google.cloud.spanner_v1 import param_types
 from google.cloud.spanner_v1.client import Client
@@ -132,6 +133,8 @@
     "TransactionSelector",
     "Type",
     "TypeCode",
+    # Custom spanner related data types
+    "JsonObject",
     # google.cloud.spanner_v1.services
     "SpannerClient",
 )
diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py
index c7cdf7aedc..fc3512f0ec 100644
--- a/google/cloud/spanner_v1/_helpers.py
+++ b/google/cloud/spanner_v1/_helpers.py
@@ -17,6 +17,7 @@
 import datetime
 import decimal
 import math
+import json
 
 import six
 
@@ -28,7 +29,7 @@
 from google.cloud._helpers import _datetime_to_rfc3339
 from google.cloud.spanner_v1 import TypeCode
 from google.cloud.spanner_v1 import ExecuteSqlRequest
-
+from google.cloud.spanner_v1 import JsonObject
 
 # Validation error messages
 NUMERIC_MAX_SCALE_ERR_MSG = (
@@ -166,6 +167,10 @@ def _make_value_pb(value):
     if isinstance(value, decimal.Decimal):
         _assert_numeric_precision_and_scale(value)
         return Value(string_value=str(value))
+    if isinstance(value, JsonObject):
+        return Value(
+            string_value=json.dumps(value, sort_keys=True, separators=(",", ":"),)
+        )
     raise ValueError("Unknown type: %s" % (value,))
 
 
diff --git a/google/cloud/spanner_v1/data_types.py b/google/cloud/spanner_v1/data_types.py
new file mode 100644
index 0000000000..305c0cb2a9
--- /dev/null
+++ b/google/cloud/spanner_v1/data_types.py
@@ -0,0 +1,25 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Custom data types for spanner."""
+
+
+class JsonObject(dict):
+    """
+    JsonObject type help format Django JSONField to compatible Cloud Spanner's
+    JSON type. Before making queries, it'll help differentiate between
+    normal parameters and JSON parameters.
+    """
+
+    pass
diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py
index 59237113e6..d9fded9c0b 100644
--- a/tests/system/test_backup_api.py
+++ b/tests/system/test_backup_api.py
@@ -400,6 +400,11 @@ def test_instance_list_backups(
     )
     expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
+    # Backup tests are failing because of timeout. As a temporary fix
+    # we are increasing db version time to current time.
+    # Read more: https://github.com/googleapis/python-spanner/issues/496
+    database_version_time = datetime.datetime.now(datetime.timezone.utc)
+
     backup1 = shared_instance.backup(
         backup_id_1,
         database=shared_database,
diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py
index f6af3c3763..2d1b4097dc 100644
--- a/tests/system/test_dbapi.py
+++ b/tests/system/test_dbapi.py
@@ -15,11 +15,11 @@
 import hashlib
 import pickle
 import pkg_resources
-
 import pytest
 
 from google.cloud import spanner_v1
 from google.cloud.spanner_dbapi.connection import connect, Connection
+from google.cloud.spanner_v1 import JsonObject
 from . import _helpers
 
 DATABASE_NAME = "dbapi-txn"
@@ -328,6 +328,45 @@ def test_DDL_autocommit(shared_instance, dbapi_database):
     conn.commit()
 
 
+@pytest.mark.skipif(_helpers.USE_EMULATOR, reason="Emulator does not support json.")
+def test_autocommit_with_json_data(shared_instance, dbapi_database):
+    """Check that DDLs in autocommit mode are immediately executed for
+    json fields."""
+    # Create table
+    conn = Connection(shared_instance, dbapi_database)
+    conn.autocommit = True
+
+    cur = conn.cursor()
+    cur.execute(
+        """
+        CREATE TABLE JsonDetails (
+            DataId     INT64 NOT NULL,
+            Details    JSON,
+        ) PRIMARY KEY (DataId)
+    """
+    )
+
+    # Insert data to table
+    cur.execute(
+        sql="INSERT INTO JsonDetails (DataId, Details) VALUES (%s, %s)",
+        args=(123, JsonObject({"name": "Jakob", "age": "26"})),
+    )
+
+    # Read back the data.
+    cur.execute("""select * from JsonDetails;""")
+    got_rows = cur.fetchall()
+
+    # Assert the response
+    assert len(got_rows) == 1
+    assert got_rows[0][0] == 123
+    assert got_rows[0][1] == '{"age":"26","name":"Jakob"}'
+
+    # Drop the table
+    cur.execute("DROP TABLE JsonDetails")
+    conn.commit()
+    conn.close()
+
+
 def test_DDL_commit(shared_instance, dbapi_database):
     """Check that DDLs in commit mode are executed on calling `commit()`."""
     conn = Connection(shared_instance, dbapi_database)
diff --git a/tests/unit/spanner_dbapi/test_parse_utils.py b/tests/unit/spanner_dbapi/test_parse_utils.py
index 4de429076e..994b02d615 100644
--- a/tests/unit/spanner_dbapi/test_parse_utils.py
+++ b/tests/unit/spanner_dbapi/test_parse_utils.py
@@ -16,6 +16,7 @@
 import unittest
 
 from google.cloud.spanner_v1 import param_types
+from google.cloud.spanner_v1 import JsonObject
 
 
 class TestParseUtils(unittest.TestCase):
@@ -333,9 +334,11 @@ def test_get_param_types(self):
         import datetime
         import decimal
 
-        from google.cloud.spanner_dbapi.parse_utils import DateStr
-        from google.cloud.spanner_dbapi.parse_utils import TimestampStr
-        from google.cloud.spanner_dbapi.parse_utils import get_param_types
+        from google.cloud.spanner_dbapi.parse_utils import (
+            DateStr,
+            TimestampStr,
+            get_param_types,
+        )
 
         params = {
             "a1": 10,
@@ -349,6 +352,7 @@ def test_get_param_types(self):
             "i1": b"bytes",
             "j1": None,
             "k1": decimal.Decimal("3.194387483193242e+19"),
+            "l1": JsonObject({"key": "value"}),
         }
         want_types = {
             "a1": param_types.INT64,
@@ -361,6 +365,7 @@ def test_get_param_types(self):
             "h1": param_types.DATE,
             "i1": param_types.BYTES,
             "k1": param_types.NUMERIC,
+            "l1": param_types.JSON,
         }
         got_types = get_param_types(params)
         self.assertEqual(got_types, want_types)

From db63aee2b15fd812d78d980bc302d9a217ca711e Mon Sep 17 00:00:00 2001
From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com>
Date: Tue, 5 Oct 2021 03:15:06 +0530
Subject: [PATCH 2/4] fix: remove database_version_time param from
 test_instance_list_backups (#609)

---
 tests/system/test_backup_api.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py
index d9fded9c0b..de521775d4 100644
--- a/tests/system/test_backup_api.py
+++ b/tests/system/test_backup_api.py
@@ -383,11 +383,7 @@ def test_multi_create_cancel_update_error_restore_errors(
 
 
 def test_instance_list_backups(
-    shared_instance,
-    shared_database,
-    second_database,
-    database_version_time,
-    backups_to_delete,
+    shared_instance, shared_database, second_database, backups_to_delete,
 ):
     # Remove un-scrubbed backups FBO count below.
     _helpers.scrub_instance_backups(shared_instance)
@@ -400,16 +396,8 @@ def test_instance_list_backups(
     )
     expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
-    # Backup tests are failing because of timeout. As a temporary fix
-    # we are increasing db version time to current time.
-    # Read more: https://github.com/googleapis/python-spanner/issues/496
-    database_version_time = datetime.datetime.now(datetime.timezone.utc)
-
     backup1 = shared_instance.backup(
-        backup_id_1,
-        database=shared_database,
-        expire_time=expire_time_1,
-        version_time=database_version_time,
+        backup_id_1, database=shared_database, expire_time=expire_time_1,
     )
 
     expire_time_2 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(

From 0404a3354ebe225f775e381eff8e43f098fe7624 Mon Sep 17 00:00:00 2001
From: larkee <31196561+larkee@users.noreply.github.com>
Date: Tue, 5 Oct 2021 12:31:36 +1300
Subject: [PATCH 3/4] test: delete referencing databases before backups (#581)

* test: delete referencing databases before backups

* fix: use proto value

Co-authored-by: larkee <larkee@users.noreply.github.com>
---
 tests/system/_helpers.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py
index 0baff62433..ffd099b996 100644
--- a/tests/system/_helpers.py
+++ b/tests/system/_helpers.py
@@ -71,9 +71,20 @@ def _has_all_ddl(database):
 retry_has_all_dll = retry.RetryInstanceState(_has_all_ddl)
 
 
+def scrub_referencing_databases(to_scrub, db_list):
+    for db_name in db_list:
+        db = to_scrub.database(db_name.split("/")[-1])
+        try:
+            retry_429_503(db.delete)()
+        except exceptions.NotFound:  # lost the race
+            pass
+
+
 def scrub_instance_backups(to_scrub):
     try:
         for backup_pb in to_scrub.list_backups():
+            # Backup cannot be deleted while referencing databases exist.
+            scrub_referencing_databases(to_scrub, backup_pb.referencing_databases)
             bkp = instance_mod.Backup.from_pb(backup_pb, to_scrub)
             try:
                 # Instance cannot be deleted while backups exist.

From aaec1db29c2faa9677174d8e27978c4a1cabb8f3 Mon Sep 17 00:00:00 2001
From: "release-please[bot]"
 <55107282+release-please[bot]@users.noreply.github.com>
Date: Tue, 5 Oct 2021 16:48:12 +0530
Subject: [PATCH 4/4] chore: release 3.11.1 (#607)

Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>
Co-authored-by: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com>
---
 CHANGELOG.md | 8 ++++++++
 setup.py     | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58c03fd129..d205608d01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,14 @@
 
 [1]: https://pypi.org/project/google-cloud-spanner/#history
 
+### [3.11.1](https://www.github.com/googleapis/python-spanner/compare/v3.11.0...v3.11.1) (2021-10-04)
+
+
+### Bug Fixes
+
+* add support for json data type ([#593](https://www.github.com/googleapis/python-spanner/issues/593)) ([bc5ddc3](https://www.github.com/googleapis/python-spanner/commit/bc5ddc3fb1eb7eff9a266fe3d1c3c8a4a6fd3763))
+* remove database_version_time param from test_instance_list_backups ([#609](https://www.github.com/googleapis/python-spanner/issues/609)) ([db63aee](https://www.github.com/googleapis/python-spanner/commit/db63aee2b15fd812d78d980bc302d9a217ca711e))
+
 ## [3.11.0](https://www.github.com/googleapis/python-spanner/compare/v3.10.0...v3.11.0) (2021-09-29)
 
 
diff --git a/setup.py b/setup.py
index 5972cab454..2a8783beef 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
 
 name = "google-cloud-spanner"
 description = "Cloud Spanner API client library"
-version = "3.11.0"
+version = "3.11.1"
 # Should be one of:
 # 'Development Status :: 3 - Alpha'
 # 'Development Status :: 4 - Beta'