diff --git a/jans-pycloudlib/docs/api/persistence/spanner.md b/jans-pycloudlib/docs/api/persistence/spanner.md deleted file mode 100644 index 36727ba6351..00000000000 --- a/jans-pycloudlib/docs/api/persistence/spanner.md +++ /dev/null @@ -1,3 +0,0 @@ -::: jans.pycloudlib.persistence.spanner.render_spanner_properties - -::: jans.pycloudlib.persistence.spanner.SpannerClient diff --git a/jans-pycloudlib/docs/api/wait.md b/jans-pycloudlib/docs/api/wait.md index c039bda71ad..0096ee357c3 100644 --- a/jans-pycloudlib/docs/api/wait.md +++ b/jans-pycloudlib/docs/api/wait.md @@ -20,10 +20,6 @@ ::: jans.pycloudlib.wait.wait_for_couchbase_conn -::: jans.pycloudlib.wait.wait_for_spanner - -::: jans.pycloudlib.wait.wait_for_spanner_conn - ::: jans.pycloudlib.wait.wait_for_sql ::: jans.pycloudlib.wait.wait_for_sql_conn diff --git a/jans-pycloudlib/jans/pycloudlib/lock/__init__.py b/jans-pycloudlib/jans/pycloudlib/lock/__init__.py index 56e18d83acb..4884995bfe3 100644 --- a/jans-pycloudlib/jans/pycloudlib/lock/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/lock/__init__.py @@ -19,7 +19,6 @@ import backoff from jans.pycloudlib.lock.couchbase_lock import CouchbaseLock -from jans.pycloudlib.lock.spanner_lock import SpannerLock from jans.pycloudlib.lock.sql_lock import SqlLock from jans.pycloudlib.utils import as_boolean from jans.pycloudlib.persistence.utils import PersistenceMapper @@ -33,13 +32,12 @@ _DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%fZ" -LockAdapter = _t.Union[SqlLock, SpannerLock, CouchbaseLock] +LockAdapter = _t.Union[SqlLock, CouchbaseLock] """Lock adapter type. Currently supports the following classes: * [SqlLock][jans.pycloudlib.lock.sql_lock.SqlLock] -* [SpannerLock][jans.pycloudlib.lock.spanner_lock.SpannerLock] * [CouchbaseLock][jans.pycloudlib.lock.couchbase_lock.CouchbaseLock] """ @@ -251,7 +249,6 @@ def adapter(self) -> LockAdapter: # noqa: D412 Supported lock adapter name: - `sql`: returns an instance of [SqlLock][jans.pycloudlib.lock.sql_lock.SqlLock] - - `spanner`: returns and instance of [SpannerLock][jans.pycloudlib.lock.spanner_lock.SpannerLock] - `couchbase`: returns and instance of [CouchbaseLock][jans.pycloudlib.lock.couchbase_lock.CouchbaseLock] """ _adapter = os.environ.get("CN_OCI_LOCK_ADAPTER") or PersistenceMapper().mapping["default"] @@ -259,9 +256,6 @@ def adapter(self) -> LockAdapter: # noqa: D412 if _adapter == "sql": return SqlLock() - if _adapter == "spanner": - return SpannerLock() - if _adapter == "couchbase": return CouchbaseLock() @@ -444,7 +438,6 @@ def release(self) -> None: # avoid implicit reexport disabled error __all__ = [ "LockManager", - "SpannerLock", "SqlLock", "CouchbaseLock", ] diff --git a/jans-pycloudlib/jans/pycloudlib/lock/spanner_lock.py b/jans-pycloudlib/jans/pycloudlib/lock/spanner_lock.py deleted file mode 100644 index 2f7951f9206..00000000000 --- a/jans-pycloudlib/jans/pycloudlib/lock/spanner_lock.py +++ /dev/null @@ -1,143 +0,0 @@ -from __future__ import annotations - -import json -import logging -import os -import typing as _t -from contextlib import suppress -from functools import cached_property - -from google.api_core.exceptions import AlreadyExists -from google.api_core.exceptions import NotFound -from google.cloud.spanner_v1 import Client -from google.cloud.spanner_v1.param_types import STRING - -from jans.pycloudlib.lock.base_lock import BaseLock - -if _t.TYPE_CHECKING: # pragma: no cover - # imported objects for function type hint, completion, etc. - # these won't be executed in runtime - from google.cloud.spanner_v1.database import Database - from google.cloud.spanner_v1.instance import Instance - -logger = logging.getLogger(__name__) - - -class SpannerLock(BaseLock): - @property - def table_name(self): - return "jansOciLock" - - @cached_property - def client(self) -> Client: - """Get an instance Spanner client object.""" - project_id = os.environ.get("GOOGLE_PROJECT_ID", "") - return Client(project=project_id) # type: ignore - - @cached_property - def instance(self) -> Instance: - """Get an instance Spanner instance object.""" - instance_id = os.environ.get("CN_GOOGLE_SPANNER_INSTANCE_ID", "") - return self.client.instance(instance_id) # type: ignore - - @cached_property - def database(self) -> Database: - """Get an instance Spanner database object.""" - database_id = os.environ.get("CN_GOOGLE_SPANNER_DATABASE_ID", "") - return self.instance.database(database_id) # type: ignore - - def _prepare_table(self): - if not self.database.table(self.table_name).exists(): - # note that JSON type is not supported by current Janssen version - # hence the jansData type is set as text - stmt = " ".join([ - f"CREATE TABLE {self.table_name}", - "(doc_id STRING(128), jansData STRING(MAX))", - "PRIMARY KEY (doc_id)", - ]) - self.database.update_ddl([stmt]) - - def get(self, key: str) -> dict[str, _t.Any]: - self._prepare_table() - - columns = ["doc_id", "jansData"] - - with self.database.snapshot() as snapshot: - result = snapshot.execute_sql( - f"SELECT * FROM {self.table_name} WHERE doc_id = @key LIMIT 1", # nosec: B608 - params={"key": key}, - param_types={"key": STRING}, - ) - with suppress(IndexError, NotFound): - row = list(result)[0] - entry = dict(zip(columns, row)) - return json.loads(entry["jansData"]) | {"name": entry["doc_id"]} - return {} - - def post(self, key: str, owner: str, ttl: float, updated_at: str) -> bool: - self._prepare_table() - - def insert_row(transaction): - return transaction.execute_update( - f"INSERT INTO {self.table_name} (doc_id, jansData) VALUES (@key, @data)", # nosec: B608 - params={ - "key": key, - "data": json.dumps({"owner": owner, "ttl": ttl, "updated_at": updated_at}), - }, - param_types={ - "key": STRING, - "data": STRING, - }, - ) - - with suppress(AlreadyExists): - created = self.database.run_in_transaction(insert_row) - return bool(created) - return False - - def put(self, key: str, owner: str, ttl: float, updated_at: str) -> bool: - self._prepare_table() - - def update_row(transaction): - return transaction.execute_update( - f"UPDATE {self.table_name} SET jansData = @data WHERE doc_id = @key", # nosec: B608 - params={ - "key": key, - "data": json.dumps({"owner": owner, "ttl": ttl, "updated_at": updated_at}), - }, - param_types={ - "key": STRING, - "data": STRING, - }, - ) - - with suppress(NotFound): - updated = self.database.run_in_transaction(update_row) - return bool(updated) - return False - - def delete(self, key: str) -> bool: - self._prepare_table() - - def delete_row(transaction): - return transaction.execute_update( - f"DELETE FROM {self.table_name} WHERE doc_id = @key", # nosec: B608 - params={"key": key}, - param_types={"key": STRING}, - ) - deleted = self.database.run_in_transaction(delete_row) - return bool(deleted) - - def connected(self) -> bool: - """Check if connection is established. - - Returns: - A boolean to indicate connection is established. - """ - cntr = 0 - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.execute_sql("SELECT 1") - with suppress(IndexError): - row = list(result)[0] - cntr = row[0] - return cntr > 0 diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py b/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py index 0230bea0c8f..a68302fd673 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/__init__.py @@ -7,8 +7,6 @@ from jans.pycloudlib.persistence.sql import render_sql_properties # noqa: F401 from jans.pycloudlib.persistence.sql import doc_id_from_dn # noqa: F401 from jans.pycloudlib.persistence.sql import SqlClient # noqa: F401 -from jans.pycloudlib.persistence.spanner import render_spanner_properties # noqa: F401 -from jans.pycloudlib.persistence.spanner import SpannerClient # noqa: F401 from jans.pycloudlib.persistence.utils import PersistenceMapper # noqa: F401 from jans.pycloudlib.persistence.utils import PERSISTENCE_TYPES # noqa: F401 from jans.pycloudlib.persistence.utils import PERSISTENCE_SQL_DIALECTS # noqa: F401 @@ -26,8 +24,6 @@ "render_sql_properties", "doc_id_from_dn", "SqlClient", - "render_spanner_properties", - "SpannerClient", "PersistenceMapper", "PERSISTENCE_TYPES", "PERSISTENCE_SQL_DIALECTS", diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/spanner.py b/jans-pycloudlib/jans/pycloudlib/persistence/spanner.py deleted file mode 100644 index 0128ed94321..00000000000 --- a/jans-pycloudlib/jans/pycloudlib/persistence/spanner.py +++ /dev/null @@ -1,464 +0,0 @@ -"""This module contains classes and functions for interacting with Google Spanner database.""" - -from __future__ import annotations - -import hashlib -import json -import logging -import os -import typing as _t -from contextlib import suppress -from functools import cached_property -from tempfile import NamedTemporaryFile - -from google.api_core.exceptions import AlreadyExists -from google.api_core.exceptions import NotFound -from google.api_core.exceptions import FailedPrecondition -from google.cloud.spanner_v1 import Client -from google.cloud.spanner_v1.keyset import KeySet -from google.cloud.spanner_v1.param_types import STRING -from ldif import LDIFParser - -from jans.pycloudlib.persistence.sql import doc_id_from_dn -from jans.pycloudlib.persistence.sql import SqlSchemaMixin -from jans.pycloudlib.utils import safe_render - -if _t.TYPE_CHECKING: # pragma: no cover - # imported objects for function type hint, completion, etc. - # these won't be executed in runtime - from google.cloud.spanner_v1.database import Database - from google.cloud.spanner_v1.instance import Instance - from jans.pycloudlib.manager import Manager - - -logger = logging.getLogger(__name__) - - -class SpannerClient(SqlSchemaMixin): - """Class to interact with Spanner database. - - The following envvars are required: - - - ``GOOGLE_APPLICATION_CREDENTIALS``: Path to JSON file contains Google credentials - - ``GOOGLE_PROJECT_ID``: (a.k.a Google project ID) - - ``CN_GOOGLE_SPANNER_INSTANCE_ID``: Spanner instance ID - - ``CN_GOOGLE_SPANNER_DATABASE_ID``: Spanner database ID - - Args: - manager: An instance of manager class. - *args: Positional arguments. - **kwargs: Keyword arguments. - """ - - def __init__(self, manager: Manager, *args: _t.Any, **kwargs: _t.Any) -> None: - self.manager = manager - self.dialect = "spanner" - - @cached_property - def client(self) -> Client: - """Get an instance Spanner client object.""" - project_id = os.environ.get("GOOGLE_PROJECT_ID", "") - return Client(project=project_id) # type: ignore - - @cached_property - def instance(self) -> Instance: - """Get an instance Spanner instance object.""" - instance_id = os.environ.get("CN_GOOGLE_SPANNER_INSTANCE_ID", "") - return self.client.instance(instance_id) # type: ignore - - @cached_property - def database(self) -> Database: - """Get an instance Spanner database object.""" - database_id = os.environ.get("CN_GOOGLE_SPANNER_DATABASE_ID", "") - return self.instance.database(database_id) # type: ignore - - @cached_property - def sub_tables(self) -> dict[str, list[list[str]]]: - """Get a mapping of subtables from pre-defined file.""" - with open("/app/static/rdbm/sub_tables.json") as f: - return json.loads(f.read()).get(self.dialect, {}) # type: ignore - - def connected(self) -> bool: - """Check whether connection is alive by executing simple query.""" - cntr = 0 - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.execute_sql("SELECT 1") - with suppress(IndexError): - row = list(result)[0] - cntr = row[0] - return cntr > 0 - - def create_table(self, table_name: str, column_mapping: dict[str, str], pk_column: str) -> None: - """Create table with its columns.""" - columns = [] - for column_name, column_type in column_mapping.items(): - column_def = f"{self.quoted_id(column_name)} {column_type}" - - if column_name == pk_column: - column_def += " NOT NULL" - columns.append(column_def) - - columns_fmt = ", ".join(columns) - pk_def = f"PRIMARY KEY ({self.quoted_id(pk_column)})" - query = f"CREATE TABLE {self.quoted_id(table_name)} ({columns_fmt}) {pk_def}" - - try: - self.database.update_ddl([query]) # type: ignore - except FailedPrecondition as exc: - if "Duplicate name in schema" in exc.args[0]: - # table exists - pass - else: - raise - - def quoted_id(self, identifier: str) -> str: - """Get quoted identifier name.""" - char = '`' - return f"{char}{identifier}{char}" - - def get_table_mapping(self) -> dict[str, dict[str, str]]: - """Get mapping of column name and type from all tables.""" - table_mapping = {} - for table in self.database.list_tables(): # type: ignore - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.execute_sql( - "select column_name, spanner_type " - "from information_schema.columns " - "where table_name = @table_name", - params={"table_name": table.table_id}, - param_types={"table_name": STRING}, - ) - table_mapping[table.table_id] = dict(result) - return table_mapping - - def insert_into(self, table_name: str, column_mapping: dict[str, _t.Any]) -> None: - """Insert a row into a table.""" - # TODO: handle ARRAY ? - def insert_rows(transaction): # type: ignore - transaction.insert( - table_name, - columns=column_mapping.keys(), - values=[column_mapping.values()] - ) - - with suppress(AlreadyExists): - self.database.run_in_transaction(insert_rows) # type: ignore - - def row_exists(self, table_name: str, id_: str) -> bool: - """Check whether a row is exist.""" - exists = False - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.read( - table=table_name, - columns=["doc_id"], - keyset=KeySet([[id_]]), # type: ignore - limit=1, - ) - with suppress(IndexError, NotFound): - row = list(result)[0] - if row: - exists = True - return exists - - def create_index(self, query: str) -> None: - """Create index using raw query.""" - try: - self.database.update_ddl([query]) # type: ignore - except FailedPrecondition as exc: - if "Duplicate name in schema" in exc.args[0]: - # table exists - pass - else: - raise - - def create_subtable( - self, - table_name: str, - sub_table_name: str, - column_mapping: dict[str, str], - pk_column: str, - sub_pk_column: str - ) -> None: - """Create sub table with its columns.""" - columns = [] - for column_name, column_type in column_mapping.items(): - column_def = f"{self.quoted_id(column_name)} {column_type}" - - if column_name == pk_column: - column_def += " NOT NULL" - columns.append(column_def) - - columns_fmt = ", ".join(columns) - pk_def = f"PRIMARY KEY ({self.quoted_id(pk_column)}, {self.quoted_id(sub_pk_column)})" - query = ", ".join([ - f"CREATE TABLE {self.quoted_id(sub_table_name)} ({columns_fmt}) {pk_def}", - f"INTERLEAVE IN PARENT {self.quoted_id(table_name)} ON DELETE CASCADE" - ]) - - try: - self.database.update_ddl([query]) # type: ignore - except FailedPrecondition as exc: - if "Duplicate name in schema" in exc.args[0]: - # table exists - pass - else: - raise - - def get(self, table_name: str, id_: str, column_names: _t.Union[list[str], None] = None) -> dict[str, _t.Any]: - """Get a row from a table with matching ID.""" - if not column_names: - # TODO: faster lookup on column names - col_names = list(self.get_table_mapping().get(table_name, {}).keys()) - else: - col_names = [] - - entry = {} - - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.read( - table=table_name, - columns=col_names, - keyset=KeySet([[id_]]), # type: ignore - limit=1, - ) - with suppress(IndexError, NotFound): - row = list(result)[0] - entry = dict(zip(col_names, row)) - return entry - - def update(self, table_name: str, id_: str, column_mapping: dict[str, _t.Any]) -> bool: - """Update a table row with matching ID.""" - # TODO: handle ARRAY ? - def update_rows(transaction): # type: ignore - # need to add primary key - column_mapping["doc_id"] = id_ - transaction.update( - table_name, - columns=column_mapping.keys(), - values=[column_mapping.values()] - ) - - modified = False - with suppress(NotFound): - self.database.run_in_transaction(update_rows) # type: ignore - modified = True - return modified - - def search(self, table_name: str, column_names: _t.Union[list[str], None] = None) -> _t.Iterator[dict[str, _t.Any]]: - """Get all rows from a table.""" - if not column_names: - # TODO: faster lookup on column names - column_names = list(self.get_table_mapping().get(table_name, {}).keys()) - - with self.database.snapshot() as snapshot: # type: ignore - result = snapshot.read( - table=table_name, - columns=column_names, - keyset=KeySet(all_=True), # type: ignore - ) - for row in result: - yield dict(zip(column_names, row)) - - def insert_into_subtable(self, table_name: str, column_mapping: dict[str, _t.Any]) -> None: - """Add new entry into subtable. - - Args: - table_name: Subtable name. - column_mapping: Key-value pairs of column name and its value. - """ - for column, value in column_mapping.items(): - if not self.column_in_subtable(table_name, column): - continue - - for item in value: - hashed = hashlib.sha256() - hashed.update(item.encode()) - dict_doc_id = hashed.digest().hex() - - self.insert_into( - f"{table_name}_{column}", - { - "doc_id": column_mapping["doc_id"], - "dict_doc_id": dict_doc_id, - column: item - }, - ) - - def _transform_value(self, key: str, values: _t.Any) -> _t.Any: - """Transform value from one to another based on its data type. - - Args: - key: Attribute name. - values: Pre-transformed values. - """ - type_ = self.sql_data_types.get(key, {}) - - if not type_: - type_ = self.sql_json_types.get(key, {}) - - if not type_: - attr_syntax = self.get_attr_syntax(key) - type_ = self.sql_data_types_mapping[attr_syntax] - - type_ = type_.get(self.dialect, {}) - data_type = type_.get("type", "") - - if data_type in ("SMALLINT", "BOOL",): - if values[0].lower() in ("1", "on", "true", "yes", "ok"): - return 1 if data_type == "SMALLINT" else True - return 0 if data_type == "SMALLINT" else False - - if data_type == "INT64": - return int(values[0]) - - if data_type in ("DATETIME(3)", "TIMESTAMP",): - dval = values[0].strip("Z") - sep = "T" - postfix = "Z" - return "{}-{}-{}{}{}:{}:{}{}{}".format( - dval[0:4], - dval[4:6], - dval[6:8], - sep, - dval[8:10], - dval[10:12], - dval[12:14], - dval[14:17], - postfix, - ) - - if data_type == "JSON": - return values - - if data_type == "ARRAY": - return values - - # fallback - return values[0] - - def _data_from_ldif(self, filename: str) -> _t.Iterator[tuple[str, dict[str, _t.Any]]]: - """Get data from parsed LDIF file. - - Args: - filename: LDIF filename. - """ - with open(filename, "rb") as fd: - parser = LDIFParser(fd) - - for dn, entry in parser.parse(): - doc_id = doc_id_from_dn(dn) - - oc = entry.get("objectClass") or entry.get("objectclass") - if oc: - if "top" in oc: - oc.remove("top") - - if len(oc) == 1 and oc[0].lower() in ("organizationalunit", "organization"): - continue - - table_name = oc[-1] - - # remove objectClass - entry.pop("objectClass", None) - entry.pop("objectclass", None) - - attr_mapping = { - "doc_id": doc_id, - "objectClass": table_name, - "dn": dn, - } - - for attr in entry: - # TODO: check if attr in sub table - value = self._transform_value(attr, entry[attr]) - attr_mapping[attr] = value - yield table_name, attr_mapping - - def create_from_ldif(self, filepath: str, ctx: dict[str, _t.Any]) -> None: - """Create entry with data loaded from an LDIF template file. - - Args: - filepath: Path to LDIF template file. - ctx: Key-value pairs of context that rendered into LDIF template file. - """ - with open(filepath) as src, NamedTemporaryFile("w+") as dst: - dst.write(safe_render(src.read(), ctx)) - # ensure rendered template is written - dst.flush() - - for table_name, column_mapping in self._data_from_ldif(dst.name): - self.insert_into(table_name, column_mapping) - self.insert_into_subtable(table_name, column_mapping) - - def column_in_subtable(self, table_name: str, column: str) -> bool: - """Check whether a subtable has certain column. - - Args: - table_name: Name of the subtable. - column: Name of the column. - """ - exists = False - - # column_mapping is a list - column_mapping = self.sub_tables.get(table_name, []) - for cm in column_mapping: - if column == cm[0]: - exists = True - break - return exists - - def delete(self, table_name: str, id_: str) -> bool: - """Delete a row from a table with matching ID.""" - deleted = False - - with self.database.batch() as batch: - batch.delete(table_name, KeySet([[id_]])) - deleted = True - return deleted - - @property - def use_simple_json(self): - """Determine whether to use simple JSON where values are stored as JSON array.""" - return True - - -def render_spanner_properties(manager: Manager, src: str, dest: str) -> None: - """Render file contains properties to connect to Spanner database. - - Args: - manager: An instance of :class:`~jans.pycloudlib.manager.Manager`. - src: Absolute path to the template. - dest: Absolute path where generated file is located. - """ - with open(src) as f: - txt = f.read() - - with open(dest, "w") as f: - if "SPANNER_EMULATOR_HOST" in os.environ: - emulator_host = os.environ.get("SPANNER_EMULATOR_HOST") - creds = f"connection.emulator-host={emulator_host}" - else: - cred_file = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") - creds = f"connection.credentials-file={cred_file}" - - rendered_txt = txt % { - "spanner_project": os.environ.get("GOOGLE_PROJECT_ID", ""), - "spanner_instance": os.environ.get("CN_GOOGLE_SPANNER_INSTANCE_ID", ""), - "spanner_database": os.environ.get("CN_GOOGLE_SPANNER_DATABASE_ID", ""), - "spanner_creds": creds, - } - f.write(rendered_txt) - - -def sync_google_credentials(manager): - path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") or "" - path_exists = os.path.isfile(path) - - if path_exists: - manager.secret.set("google_credentials", path) - - # make sure file always exists (unless path is invalid and secret is empty) - if not path_exists and (contents := manager.secret.get("google_credentials")): - with suppress(FileNotFoundError): - with open(path, "w") as f: - f.write(contents) diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/sql.py b/jans-pycloudlib/jans/pycloudlib/persistence/sql.py index 0adaa5b5430..c05c03aceae 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/sql.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/sql.py @@ -211,7 +211,6 @@ def sql_json_types(self): json_types[attr] = { "mysql": {"type": "JSON"}, "pgsql": {"type": "JSONB"}, - "spanner": {"type": "ARRAY"}, } return json_types diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/utils.py b/jans-pycloudlib/jans/pycloudlib/persistence/utils.py index 01b20d44a02..dec38a87630 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/utils.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/utils.py @@ -58,7 +58,6 @@ def render_base_properties(src: str, dest: str) -> None: PERSISTENCE_TYPES = ( "couchbase", "sql", - "spanner", "hybrid", ) """Supported persistence types.""" @@ -127,7 +126,6 @@ class PersistenceMapper: os.environ["CN_PERSISTENCE_TYPE"] = "hybrid" os.environ["CN_HYBRID_MAPPING"] = json.loads({ "default": "sql", - "user": "spanner", "site": "sql", "cache": "sql", "token": "sql", @@ -144,7 +142,7 @@ class PersistenceMapper: ```py { "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "sql", @@ -168,7 +166,7 @@ def mapping(self) -> dict[str, str]: ```py { "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "sql", @@ -190,9 +188,8 @@ def groups(self) -> dict[str, list[str]]: ```py { - "sql": ["cache", "default", "session", "site"], + "sql": ["cache", "default", "session", "site", "token"], "couchbase": ["user"], - "spanner": ["token"], } ``` """ @@ -209,9 +206,8 @@ def groups_with_rdn(self) -> dict[str, list[str]]: ```py { - "sql": ["cache", "", "sessions", "link"], + "sql": ["cache", "", "sessions", "link", "tokens"], "couchbase": ["people, groups, authorizations"], - "spanner": ["tokens"], } ``` """ diff --git a/jans-pycloudlib/jans/pycloudlib/validators.py b/jans-pycloudlib/jans/pycloudlib/validators.py index cbcc8c53bbe..dd714af9f2c 100644 --- a/jans-pycloudlib/jans/pycloudlib/validators.py +++ b/jans-pycloudlib/jans/pycloudlib/validators.py @@ -12,7 +12,6 @@ def validate_persistence_type(type_: str) -> None: - `couchbase` - `hybrid` - - `spanner` - `sql` Args: diff --git a/jans-pycloudlib/jans/pycloudlib/wait.py b/jans-pycloudlib/jans/pycloudlib/wait.py index 3bdc00d52b2..c3f3cff4317 100644 --- a/jans-pycloudlib/jans/pycloudlib/wait.py +++ b/jans-pycloudlib/jans/pycloudlib/wait.py @@ -13,7 +13,6 @@ from jans.pycloudlib.persistence.couchbase import id_from_dn from jans.pycloudlib.persistence.sql import SqlClient from jans.pycloudlib.persistence.sql import doc_id_from_dn -from jans.pycloudlib.persistence.spanner import SpannerClient from jans.pycloudlib.utils import as_boolean from jans.pycloudlib.persistence.utils import PersistenceMapper @@ -256,46 +255,6 @@ def wait_for_sql(manager: Manager, **kwargs: _t.Any) -> None: raise WaitError("SQL backend is not fully initialized") -@retry_on_exception -def wait_for_spanner_conn(manager: Manager, **kwargs: _t.Any) -> None: - """Wait for readiness/liveness of an Spanner database connection. - - Args: - manager: An instance of manager class. - **kwargs: Arbitrary keyword arguments (see Other Parameters section, if any). - """ - # checking connection - init = SpannerClient(manager).connected() - if not init: - raise WaitError("Spanner backend is unreachable") - - -@retry_on_exception -def wait_for_spanner(manager: Manager, **kwargs: _t.Any) -> None: - """Wait for readiness/liveness of an Spanner database. - - Args: - manager: An instance of manager class. - **kwargs: Arbitrary keyword arguments (see Other Parameters section, if any). - """ - search_mapping = { - "default": (doc_id_from_dn("ou=jans-auth,ou=configuration,o=jans"), "jansAppConf"), - "user": (doc_id_from_dn(_ADMIN_GROUP_DN), "jansGrp"), - } - - client = SpannerClient(manager) - try: - # get the first data key - key = PersistenceMapper().groups().get("spanner", [])[0] - doc_id, table_name = search_mapping[key] - init = client.row_exists(table_name, doc_id) - except (IndexError, KeyError): - init = client.connected() - - if not init: - raise WaitError("Spanner backend is not fully initialized") - - WaitCallback = _t.TypedDict("WaitCallback", { "func": _t.Callable[..., None], "kwargs": dict[str, _t.Any], @@ -315,8 +274,6 @@ def wait_for(manager: Manager, deps: _t.Union[list[str], None] = None) -> None: - `secret_conn` - `sql` - `sql_conn` - - `spanner` - - `spanner_conn` Args: manager: An instance of manager class. @@ -350,8 +307,6 @@ def wait_for(manager: Manager, deps: _t.Union[list[str], None] = None) -> None: }, "sql_conn": {"func": wait_for_sql_conn, "kwargs": {"label": "SQL"}}, "sql": {"func": wait_for_sql, "kwargs": {"label": "SQL"}}, - "spanner_conn": {"func": wait_for_spanner_conn, "kwargs": {"label": "Spanner"}}, - "spanner": {"func": wait_for_spanner, "kwargs": {"label": "Spanner"}}, } dependencies = deps or [] diff --git a/jans-pycloudlib/mkdocs.yml b/jans-pycloudlib/mkdocs.yml index 5195ff562ed..f20a959b80d 100644 --- a/jans-pycloudlib/mkdocs.yml +++ b/jans-pycloudlib/mkdocs.yml @@ -48,7 +48,6 @@ nav: - "Secret": api/secret.md - "Persistence": - "Couchbase": api/persistence/couchbase.md - - "Spanner": api/persistence/spanner.md - "SQL": api/persistence/sql.md - "Hybrid": api/persistence/hybrid.md - "Utilities": api/persistence/utils.md diff --git a/jans-pycloudlib/setup.py b/jans-pycloudlib/setup.py index 53c7e54ade8..f9bf0fb718e 100644 --- a/jans-pycloudlib/setup.py +++ b/jans-pycloudlib/setup.py @@ -47,7 +47,6 @@ def find_version(*file_paths): "pymysql>=1.0.2", "sqlalchemy>=1.3,<1.4", "psycopg2>=2.8.6", - "google-cloud-spanner>=3.3.0", "Click>=6.7", "ldif>=4.1.1", # handle CVE-2022-36087 diff --git a/jans-pycloudlib/tests/conftest.py b/jans-pycloudlib/tests/conftest.py index ff6e12b02c2..2fc968e5de5 100644 --- a/jans-pycloudlib/tests/conftest.py +++ b/jans-pycloudlib/tests/conftest.py @@ -111,16 +111,6 @@ def google_creds(tmpdir): yield creds -@pytest.fixture -def spanner_client(gmanager, monkeypatch, google_creds): - from jans.pycloudlib.persistence.spanner import SpannerClient - - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(google_creds)) - - client = SpannerClient(gmanager) - yield client - - @pytest.fixture def sql_client(gmanager): from jans.pycloudlib.persistence.sql import SqlClient diff --git a/jans-pycloudlib/tests/test_persistence.py b/jans-pycloudlib/tests/test_persistence.py index 3d25ae41b72..9269106c2d3 100644 --- a/jans-pycloudlib/tests/test_persistence.py +++ b/jans-pycloudlib/tests/test_persistence.py @@ -414,18 +414,17 @@ def test_resolve_hybrid_storages(monkeypatch): monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ "default": "sql", - "user": "spanner", + "user": "sql", "site": "couchbase", "cache": "sql", "token": "sql", "session": "sql", })) expected = { - "storages": "couchbase, spanner, sql", + "storages": "couchbase, sql", "storage.default": "sql", "storage.couchbase.mapping": "link", - "storage.spanner.mapping": "people, groups, authorizations", - "storage.sql.mapping": "cache, tokens, sessions", + "storage.sql.mapping": "people, groups, authorizations, cache, tokens, sessions", } mapper = PersistenceMapper() assert resolve_hybrid_storages(mapper) == expected @@ -442,17 +441,16 @@ def test_render_hybrid_properties(monkeypatch, tmpdir): "user": "couchbase", "site": "sql", "cache": "sql", - "token": "spanner", + "token": "sql", "session": "sql", }) ) expected = """ -storages: couchbase, spanner, sql +storages: couchbase, sql storage.default: sql storage.couchbase.mapping: people, groups, authorizations -storage.spanner.mapping: tokens -storage.sql.mapping: link, cache, sessions +storage.sql.mapping: link, cache, tokens, sessions """.strip() dest = tmpdir.join("jans-hybrid.properties") @@ -647,95 +645,6 @@ def test_sql_opendj_attr_types(monkeypatch): assert SqlSchemaMixin().opendj_attr_types == json.loads(types_str) -# ======= -# SPANNER -# ======= - - -def test_render_spanner_properties(monkeypatch, tmpdir, gmanager, google_creds): - from jans.pycloudlib.persistence.spanner import render_spanner_properties - - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(google_creds)) - monkeypatch.setenv("GOOGLE_PROJECT_ID", "testing-project") - monkeypatch.setenv("CN_GOOGLE_SPANNER_INSTANCE_ID", "testing-instance") - monkeypatch.setenv("CN_GOOGLE_SPANNER_DATABASE_ID", "testing-db") - - tmpl = """ -connection.project=%(spanner_project)s -connection.instance=%(spanner_instance)s -connection.database=%(spanner_database)s -%(spanner_creds)s -""".strip() - - expected = """ -connection.project=testing-project -connection.instance=testing-instance -connection.database=testing-db -connection.credentials-file={} -""".format(str(google_creds)).strip() - - src = tmpdir.join("jans-spanner.properties.tmpl") - src.write(tmpl) - dest = tmpdir.join("jans-spanner.properties") - - render_spanner_properties(gmanager, str(src), str(dest)) - assert dest.read() == expected - - -def test_render_spanner_properties_emulator(monkeypatch, tmpdir, gmanager): - from jans.pycloudlib.persistence.spanner import render_spanner_properties - - monkeypatch.setenv("SPANNER_EMULATOR_HOST", "localhost:9010") - monkeypatch.setenv("GOOGLE_PROJECT_ID", "testing-project") - monkeypatch.setenv("CN_GOOGLE_SPANNER_INSTANCE_ID", "testing-instance") - monkeypatch.setenv("CN_GOOGLE_SPANNER_DATABASE_ID", "testing-db") - - tmpl = """ -connection.project=%(spanner_project)s -connection.instance=%(spanner_instance)s -connection.database=%(spanner_database)s -%(spanner_creds)s -""".strip() - - expected = """ -connection.project=testing-project -connection.instance=testing-instance -connection.database=testing-db -connection.emulator-host=localhost:9010 -""".strip() - - src = tmpdir.join("jans-spanner.properties.tmpl") - src.write(tmpl) - dest = tmpdir.join("jans-spanner.properties") - - render_spanner_properties(gmanager, str(src), str(dest)) - assert dest.read() == expected - - -def test_spanner_quoted_id(spanner_client): - assert spanner_client.quoted_id("random") == "`random`" - - -def test_spanner_sub_tables(monkeypatch, spanner_client): - monkeypatch.setattr(BUILTINS_OPEN, lambda p: StringIO("{}")) - assert isinstance(spanner_client.sub_tables, dict) - - -def test_spanner_client_prop(spanner_client): - from google.cloud.spanner_v1.client import Client - assert isinstance(spanner_client.client, Client) - - -def test_spanner_instance_prop(spanner_client): - from google.cloud.spanner_v1.instance import Instance - assert isinstance(spanner_client.instance, Instance) - - -def test_spanner_database_prop(spanner_client): - from google.cloud.spanner_v1.database import Database - assert isinstance(spanner_client.database, Database) - - # ===== # utils # ===== @@ -744,7 +653,6 @@ def test_spanner_database_prop(spanner_client): @pytest.mark.parametrize("type_", [ "couchbase", "sql", - "spanner", ]) def test_persistence_mapper_mapping(monkeypatch, type_): from jans.pycloudlib.persistence import PersistenceMapper @@ -766,7 +674,7 @@ def test_persistence_mapper_hybrid_mapping(monkeypatch): mapping = { "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "couchbase", @@ -783,8 +691,8 @@ def test_persistence_mapper_hybrid_mapping(monkeypatch): "[]", "{}", # empty dict {"user": "sql"}, # missing remaining keys - {"default": "sql", "user": "spanner", "cache": "sql", "site": "couchbase", "token": "sql", "session": "random"}, # invalid type - {"default": "sql", "user": "spanner", "cache": "sql", "site": "couchbase", "token": "sql", "foo": "sql"}, # invalid key + {"default": "sql", "user": "sql", "cache": "sql", "site": "couchbase", "token": "sql", "session": "random"}, # invalid type + {"default": "sql", "user": "sql", "cache": "sql", "site": "couchbase", "token": "sql", "foo": "sql"}, # invalid key ]) def test_persistence_mapper_validate_hybrid_mapping(monkeypatch, mapping): from jans.pycloudlib.persistence.utils import PersistenceMapper @@ -802,7 +710,7 @@ def test_persistence_mapper_groups(monkeypatch): monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "couchbase", @@ -811,8 +719,7 @@ def test_persistence_mapper_groups(monkeypatch): groups = { "couchbase": ["token"], - "spanner": ["user"], - "sql": ["default", "site", "cache", "session"], + "sql": ["default", "user", "site", "cache", "session"], } assert PersistenceMapper().groups() == groups @@ -823,7 +730,7 @@ def test_persistence_mapper_groups_rdn(monkeypatch): monkeypatch.setenv("CN_PERSISTENCE_TYPE", "hybrid") monkeypatch.setenv("CN_HYBRID_MAPPING", json.dumps({ "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "couchbase", @@ -832,8 +739,7 @@ def test_persistence_mapper_groups_rdn(monkeypatch): groups = { "couchbase": ["tokens"], - "spanner": ["people, groups, authorizations"], - "sql": ["", "link", "cache", "sessions"], + "sql": ["", "people, groups, authorizations", "link", "cache", "sessions"], } assert PersistenceMapper().groups_with_rdn() == groups diff --git a/jans-pycloudlib/tests/test_validators.py b/jans-pycloudlib/tests/test_validators.py index cefa59a276b..be5040ac38b 100644 --- a/jans-pycloudlib/tests/test_validators.py +++ b/jans-pycloudlib/tests/test_validators.py @@ -5,7 +5,6 @@ "couchbase", "hybrid", "sql", - "spanner", ]) def test_validate_persistence_type(type_): from jans.pycloudlib.validators import validate_persistence_type diff --git a/jans-pycloudlib/tests/test_wait.py b/jans-pycloudlib/tests/test_wait.py index ec9d932062a..3c4e9ca2197 100644 --- a/jans-pycloudlib/tests/test_wait.py +++ b/jans-pycloudlib/tests/test_wait.py @@ -204,63 +204,12 @@ def test_wait_for_sql_conn(monkeypatch, gmanager): wait_for_sql_conn(gmanager) -def test_wait_for_spanner(monkeypatch, gmanager): - from jans.pycloudlib.wait import wait_for_spanner - - monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") - - monkeypatch.setattr( - "jans.pycloudlib.persistence.spanner.SpannerClient.row_exists", - lambda cls, t, i: False - ) - - with pytest.raises(Exception): - wait_for_spanner(gmanager) - - -def test_wait_for_spanner_no_search_mapping(monkeypatch, gmanager): - from jans.pycloudlib.wait import wait_for_spanner - - monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") - - monkeypatch.setattr( - _PERSISTENCE_MAPPER_GROUP_FUNC, - lambda cls: {"spanner": ["random"]} - ) - - monkeypatch.setattr( - "jans.pycloudlib.persistence.spanner.SpannerClient.connected", - lambda cls: False - ) - - with pytest.raises(Exception): - wait_for_spanner(gmanager) - - -def test_wait_for_spanner_conn(monkeypatch, gmanager): - from jans.pycloudlib.wait import wait_for_spanner_conn - - monkeypatch.setenv("CN_WAIT_MAX_TIME", "0") - monkeypatch.setenv("CN_PERSISTENCE_TYPE", "spanner") - - monkeypatch.setattr( - "jans.pycloudlib.persistence.spanner.SpannerClient.connected", - lambda cls: False - ) - - with pytest.raises(Exception): - wait_for_spanner_conn(gmanager) - - _WAIT_FOR_FUNC = "jans.pycloudlib.wait.wait_for" @pytest.mark.parametrize("persistence_type, deps", [ ("couchbase", ["couchbase"]), ("sql", ["sql"]), - ("spanner", ["spanner"]), ]) def test_wait_for_persistence(monkeypatch, gmanager, persistence_type, deps): from jans.pycloudlib.wait import wait_for_persistence @@ -280,7 +229,7 @@ def test_wait_for_persistence_hybrid(monkeypatch, gmanager): "CN_HYBRID_MAPPING", json.dumps({ "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "couchbase", @@ -290,13 +239,12 @@ def test_wait_for_persistence_hybrid(monkeypatch, gmanager): with patch(_WAIT_FOR_FUNC, autospec=True) as patched: wait_for_persistence(gmanager) - patched.assert_called_with(gmanager, ["couchbase", "spanner", "sql"]) + patched.assert_called_with(gmanager, ["couchbase", "sql"]) @pytest.mark.parametrize("persistence_type, deps", [ ("couchbase", ["couchbase_conn"]), ("sql", ["sql_conn"]), - ("spanner", ["spanner_conn"]), ]) def test_wait_for_persistence_conn(monkeypatch, gmanager, persistence_type, deps): from jans.pycloudlib.wait import wait_for_persistence_conn @@ -316,7 +264,7 @@ def test_wait_for_persistence_conn_hybrid(monkeypatch, gmanager): "CN_HYBRID_MAPPING", json.dumps({ "default": "sql", - "user": "spanner", + "user": "sql", "site": "sql", "cache": "sql", "token": "couchbase", @@ -326,7 +274,7 @@ def test_wait_for_persistence_conn_hybrid(monkeypatch, gmanager): with patch(_WAIT_FOR_FUNC, autospec=True) as patched: wait_for_persistence_conn(gmanager) - patched.assert_called_with(gmanager, ["couchbase_conn", "spanner_conn", "sql_conn"]) + patched.assert_called_with(gmanager, ["couchbase_conn", "sql_conn"]) def test_wait_for(gmanager):