From 07eb3712d8a11d7f0781f760576911f035df2c12 Mon Sep 17 00:00:00 2001 From: Andrey Popp <8mayday@gmail.com> Date: Fri, 10 Jul 2020 13:28:46 +0300 Subject: [PATCH] deploy: add support for managing SQL views This commit takes a stab at implementing support for managing SQL views with rex.deploy. The example usage: ``` - view: persons definition: | select name, email from user - view: persons definition: | select name, email from user union select name, email from patient - view: persons present: false ``` I've been trying to follow the code for tables but of course making this specific to the views. Some notes/design questions appeared: - How things are introspected has changed, previously both tables and views were loaded as TableImage objects. Now we load views as ViewImage objects and we DON'T load view columns. I suspect this will make a different when querying views with HTSQL? - When building a model out of facts we need to check for name clashes between tables and views as they share the same namespace in the database. - Should we raise an error in case the same named view appears with different definitions? Or should we instead override the former with the latter (dangerous)? - Right now we load view definition with `pg_get_viewdef` function but I'm not sure if it is suitable for comparison with view definition in facts as db can probably mangle the original SQL. We should probably store view definition in a COMMENT in a database. - Do we need to support view defined with HTSQL? That would be cool. - Would be nice to be able to specify a pk for a view (required for HTSQL to query such view). - We need to track dependencies: As we use SQL for view definitions we might need to use SQL parser for that. I'm thinking of using https://pglast.readthedocs.io/en/latest/ which wraps PostgreSQL's parser. If we want to add HTSQL based view then we can use HTSQL to infer the dependencies. - view -> table, why? - So we can require to drop view before we drop the table. It is implemented for linked tables? Would be nice as it will allow to validate migrations w/o accessing a database. - view -> view, why? - so we can require to drop dependent views before we drop the view - so we can change definition of a view with dependents (in this case we need to drop and re-create a view and its dependents) --- src/rex.deploy/src/rex/deploy/image.py | 55 +++++++++ src/rex.deploy/src/rex/deploy/introspect.py | 17 ++- src/rex.deploy/src/rex/deploy/meta.py | 15 ++- src/rex.deploy/src/rex/deploy/model.py | 103 +++++++++++++++- src/rex.deploy/src/rex/deploy/sql.py | 27 +++++ src/rex.deploy/src/rex/deploy/table.py | 124 +++++++++++++++++++- src/rex.deploy/test/test_view.rst | 104 ++++++++++++++++ 7 files changed, 441 insertions(+), 4 deletions(-) create mode 100644 src/rex.deploy/test/test_view.rst diff --git a/src/rex.deploy/src/rex/deploy/image.py b/src/rex.deploy/src/rex/deploy/image.py index d90f7c277..a42faee36 100644 --- a/src/rex.deploy/src/rex/deploy/image.py +++ b/src/rex.deploy/src/rex/deploy/image.py @@ -6,6 +6,7 @@ from .sql import (sql_create_schema, sql_drop_schema, sql_rename_schema, sql_create_extension, sql_drop_extension, sql_comment_on_schema, sql_create_table, sql_drop_table, sql_rename_table, + sql_create_view, sql_drop_view, sql_rename_view, sql_comment_on_view, sql_comment_on_table, sql_define_column, sql_add_column, sql_drop_column, sql_rename_column, sql_copy_column, sql_set_column_type, sql_set_column_not_null, sql_set_column_default, @@ -329,6 +330,10 @@ def add_table(self, name, is_unlogged=False): """Adds a table.""" return TableImage(self, name, is_unlogged=is_unlogged) + def add_view(self, name, definition): + """Adds a view.""" + return ViewImage(self, name, definition) + def add_index(self, name, table, columns): """Adds an index.""" return IndexImage(self, name, table, columns) @@ -370,6 +375,14 @@ def create_table(self, name, definitions, is_unlogged=False): table.add_column(column_name, type, is_not_null, default) return table + def create_view(self, name, definition): + """Creates a view with the given definition.""" + qname = (self.name, name) + sql = sql_create_view(qname, definition) + self.cursor.execute(sql) + view = self.add_view(name, definition=definition) + return view + def create_index(self, name, table, columns): """Creates an index.""" column_names = [column.name for column in columns] @@ -880,6 +893,48 @@ def drop(self): self.remove() +class ViewImage(NamespacedImage): + """Database view.""" + + __slots__ = ('comment', 'definition') + + def __init__(self, schema, name, definition): + super(NamespacedImage, self).__init__(schema, name) + self.place(schema.tables) + schema.link(self) + #: View comment. + self.comment = None + #: View definition. + self.definition = definition + + def set_comment(self, comment): + """Sets the comment.""" + self.comment = comment + return self + + def alter_name(self, name): + """Renames the view.""" + if self.name == name: + return self + sql = sql_rename_view(self.qname, name) + self.cursor.execute(sql) + return self.set_name(name) + + def alter_comment(self, comment): + """Updates the view comment.""" + if self.comment == comment: + return self + sql = sql_comment_on_view(self.qname, comment) + self.cursor.execute(sql) + return self.set_comment(comment) + + def drop(self): + """Drops the view.""" + sql = sql_drop_view(self.qname) + self.cursor.execute(sql) + self.remove() + + class ColumnImage(NamedImage): """Database column.""" diff --git a/src/rex.deploy/src/rex/deploy/introspect.py b/src/rex.deploy/src/rex/deploy/introspect.py index 0145a2fbb..56ad2ae35 100644 --- a/src/rex.deploy/src/rex/deploy/introspect.py +++ b/src/rex.deploy/src/rex/deploy/introspect.py @@ -93,7 +93,7 @@ def introspect(cursor): cursor.execute(""" SELECT c.oid, c.relnamespace, c.relname, c.relpersistence FROM pg_catalog.pg_class c - WHERE c.relkind IN ('r', 'v') AND + WHERE c.relkind IN ('r') AND HAS_TABLE_PRIVILEGE(c.oid, 'SELECT') ORDER BY c.relnamespace, c.relname """) @@ -103,12 +103,27 @@ def introspect(cursor): table = schema.add_table(relname, is_unlogged=is_unlogged) class_by_oid[oid] = table_by_oid[oid] = table + # Extract views. + cursor.execute(""" + SELECT c.oid, c.relnamespace, c.relname, pg_get_viewdef(c.oid) + FROM pg_catalog.pg_class c + WHERE c.relkind IN ('v') AND + HAS_TABLE_PRIVILEGE(c.oid, 'SELECT') + ORDER BY c.relnamespace, c.relname + """) + for oid, relnamespace, relname, definition in cursor.fetchall(): + schema = schema_by_oid[relnamespace] + table = schema.add_view(relname, definition) + class_by_oid[oid] = table_by_oid[oid] = table + # Extract columns. column_by_num = {} cursor.execute(""" SELECT a.attrelid, a.attnum, a.attname, a.atttypid, a.atttypmod, a.attnotnull, a.atthasdef, a.attisdropped FROM pg_catalog.pg_attribute a + JOIN pg_catalog.pg_class c ON a.attrelid = c.oid + WHERE c.relkind IN ('r') ORDER BY a.attrelid, a.attnum """) for (attrelid, attnum, attname, atttypid, diff --git a/src/rex.deploy/src/rex/deploy/meta.py b/src/rex.deploy/src/rex/deploy/meta.py index 4b1b4ca77..85edd3a4e 100644 --- a/src/rex.deploy/src/rex/deploy/meta.py +++ b/src/rex.deploy/src/rex/deploy/meta.py @@ -7,7 +7,7 @@ Location, set_location, UStrVal, UChoiceVal, MaybeVal, SeqVal, RecordVal, Error) from .fact import LabelVal, TitleVal, AliasVal, AliasSpec, FactDumper -from .image import TableImage, ColumnImage, UniqueKeyImage +from .image import TableImage, ViewImage, ColumnImage, UniqueKeyImage import operator import collections import yaml @@ -133,6 +133,17 @@ class TableMeta(Meta): ] +class ViewMeta(Meta): + """View metadata.""" + + __slots__ = () + + fields = [ + ('label', LabelVal, None), + ('title', TitleVal, None), + ] + + class ColumnMeta(Meta): """Column metadata.""" @@ -162,6 +173,8 @@ def uncomment(image): """ if isinstance(image, TableImage): return TableMeta.parse(image.comment) + if isinstance(image, ViewImage): + return ViewMeta.parse(image.comment) elif isinstance(image, ColumnImage): return ColumnMeta.parse(image.comment) elif isinstance(image, UniqueKeyImage) and image.is_primary: diff --git a/src/rex.deploy/src/rex/deploy/model.py b/src/rex.deploy/src/rex/deploy/model.py index cb4fff73b..56f72bd27 100644 --- a/src/rex.deploy/src/rex/deploy/model.py +++ b/src/rex.deploy/src/rex/deploy/model.py @@ -12,7 +12,7 @@ plpgsql_primary_key_procedure, plpgsql_integer_random_key, plpgsql_text_random_key, plpgsql_integer_offset_key, plpgsql_text_offset_key, plpgsql_text_uuid_key) -from .image import (TableImage, ColumnImage, UniqueKeyImage, CASCADE, +from .image import (TableImage, ColumnImage, ViewImage, UniqueKeyImage, CASCADE, SET_DEFAULT, BEFORE, AFTER, INSERT, INSERT_UPDATE_DELETE) from .cluster import Cluster, get_cluster import datetime @@ -257,12 +257,24 @@ def table(self, label): """ return TableModel.find(self, label) + def view(self, label): + """ + Finds the view by name. + """ + return ViewModel.find(self, label) + def build_table(self, **kwds): """ Creates a new table. """ return TableModel.do_build(self, **kwds) + def build_view(self, **kwds): + """ + Creates a new view. + """ + return ViewModel.do_build(self, **kwds) + def facts(self): """ Returns a list of facts that reproduce the schema. @@ -334,6 +346,8 @@ def find(cls, schema, label): # Finds a table by name. names = cls.names(label) image = schema.image.tables.get(names.name) + if not isinstance(image, TableImage): + image = None return schema(image) @classmethod @@ -578,6 +592,93 @@ def fact(self, with_related=False): related=related) +class ViewModel(Model): + """ + Wraps a database view. + """ + + __slots__ = ('label', 'title', 'definition') + + is_table = False + + properties = ['label', 'title', 'definition'] + + class names: + # Derives names for database objects and the view title. + + __slots__ = ('label', 'title', 'name') + + def __init__(self, label): + self.label = label + self.title = label_to_title(label) + self.name = mangle(label) + + @classmethod + def recognizes(cls, schema, image): + # Verifies if the database object is a table. + if not isinstance(image, ViewImage): + return False + # We expect the table belongs to the `public` schema. + if image.schema is not schema.image: + return False + return True + + @classmethod + def find(cls, schema, label): + # Finds a table by name. + names = cls.names(label) + image = schema.image.tables.get(names.name) + if not isinstance(image, ViewImage): + image = None + return schema(image) + + @classmethod + def do_build(cls, schema, label, definition, title=None): + # Builds a view. + names = cls.names(label) + image = schema.image.create_view(names.name, definition) + # Save the label and the title if necessary. + saved_label = label if label != names.name else None + saved_title = title if title != names.title else None + meta = uncomment(image) + if meta.update(label=saved_label, title=saved_title): + image.alter_comment(meta.dump()) + return cls(schema, image) + + def __init__(self, schema, image): + super(ViewModel, self).__init__(schema, image) + assert isinstance(image, ViewImage) + # Extract entity properties. + meta = uncomment(image) + self.label = meta.label or image.name + self.title = meta.title + self.definition = image.definition + + def do_modify(self, label, title, definition): + # Updates the state of the view entity. + # Refresh names. + names = self.names(label) + self.image.alter_name(names.name) + # Update saved label and title. + meta = uncomment(self.image) + saved_label = label if label != names.name else None + saved_title = title if title != names.title else None + if meta.update(label=saved_label, title=saved_title): + self.image.alter_comment(meta.dump()) + + def do_erase(self): + # Drops the entity. + if self.image: + self.image.drop() + + def fact(self): + from .table import ViewFact + return ViewFact( + self.label, + definition=self.image.definition, + title=self.title) + + class ColumnModel(Model): """ Wraps a table column. diff --git a/src/rex.deploy/src/rex/deploy/sql.py b/src/rex.deploy/src/rex/deploy/sql.py index 64269841b..d6a48983f 100644 --- a/src/rex.deploy/src/rex/deploy/sql.py +++ b/src/rex.deploy/src/rex/deploy/sql.py @@ -288,6 +288,33 @@ def sql_comment_on_table(qname, text): COMMENT ON TABLE {{ qname|qn }} IS {{ text|v }}; """ +@sql_template +def sql_create_view(qname, definition): + """ + CREATE VIEW {{ qname|qn }} AS ({{ definition }}); + """ + + +@sql_template +def sql_drop_view(qname): + """ + DROP VIEW {{ qname|qn }}; + """ + + +@sql_template +def sql_rename_view(qname, new_name): + """ + ALTER VIEW {{ qname|qn }} RENAME TO {{ new_name|n }}; + """ + + +@sql_template +def sql_comment_on_view(qname, text): + """ + COMMENT ON VIEW {{ qname|qn }} IS {{ text|v }}; + """ + @sql_template def sql_define_column(name, type_qname, is_not_null, default=None): diff --git a/src/rex.deploy/src/rex/deploy/table.py b/src/rex.deploy/src/rex/deploy/table.py index 986bdc7af..c623b4811 100644 --- a/src/rex.deploy/src/rex/deploy/table.py +++ b/src/rex.deploy/src/rex/deploy/table.py @@ -3,7 +3,7 @@ # -from rex.core import Error, BoolVal, SeqVal, OneOrSeqVal, locate +from rex.core import Error, BoolVal, StrVal, SeqVal, OneOrSeqVal, locate from .fact import Fact, FactVal, LabelVal, TitleVal from .model import model import collections @@ -173,3 +173,125 @@ def __call__(self, driver): driver(self.related) +class ViewFact(Fact): + """ + Describes a view. + + `label`: ``unicode`` + The name of the view. + `former_labels`: [``unicode``] + Names that the view may have had in the past. + `title`: ``unicode`` or ``None`` + The title of the view. If not set, generated from the label. + `is_present`: ``bool`` + Indicates whether the view exists in the database. + `related`: [:class:`Fact`] or ``None`` + Facts to be deployed when the view is deployed. Could be specified + only when ``is_present`` is ``True``. + `definition`: ``unicode`` + SQL definition of the view. + """ + + fields = [ + ('view', LabelVal), + ('definition', StrVal(), None), + ('was', OneOrSeqVal(LabelVal), None), + ('title', TitleVal, None), + ('present', BoolVal, True), + ] + + @classmethod + def build(cls, driver, spec): + if not spec.present: + for field in ['was', 'title', 'definition']: + if getattr(spec, field) is not None: + raise Error("Got unexpected clause:", field) + label = spec.view + definition = spec.definition + is_present = spec.present + if isinstance(spec.was, list): + former_labels = spec.was + elif spec.was: + former_labels = [spec.was] + else: + former_labels = [] + title = spec.title + after = [] + return cls(label, former_labels=former_labels, + title=title, definition=definition, + is_present=is_present) + + def __init__(self, label, definition=None, former_labels=[], + title=None, is_present=True): + # Validate input constraints. + assert isinstance(label, str) and len(label) > 0 + assert isinstance(is_present, bool) + if is_present: + assert (isinstance(former_labels, list) and + all(isinstance(former_label, str) + for former_label in former_labels)) + assert (title is None or + (isinstance(title, str) and len(title) > 0)) + else: + assert former_labels == [] + assert title is None + self.label = label + self.definition = definition + self.former_labels = former_labels + self.title = title + self.is_present = is_present + + def __repr__(self): + args = [] + args.append(repr(self.label)) + if self.definition: + args.append("definition=%r" % self.definition) + if self.former_labels: + args.append("former_labels=%r" % self.former_labels) + if self.title is not None: + args.append("title=%r" % self.title) + if not self.is_present: + args.append("is_present=%r" % self.is_present) + return "%s(%s)" % (self.__class__.__name__, ", ".join(args)) + + def to_yaml(self, full=True): + mapping = collections.OrderedDict() + mapping['view'] = self.label + if self.former_labels: + mapping['was'] = self.former_labels + if self.title is not None: + mapping['title'] = self.title + if self.is_present is False: + mapping['present'] = self.is_present + if full and self.definition: + mapping['definition'] = self.definition + return mapping + + def __call__(self, driver): + schema = model(driver) + view = schema.view(self.label) + if not view: + for former_label in self.former_labels: + view = schema.view(former_label) + if view: + break + if self.is_present: + if view: + if self.definition is None or view.definition == self.definition: + view.modify( + label=self.label, + title=self.title) + else: + view.erase() + schema.build_view( + label=self.label, + definition=self.definition, + title=self.title) + else: + schema.build_view( + label=self.label, + definition=self.definition, + title=self.title) + else: + if view: + view.erase() diff --git a/src/rex.deploy/test/test_view.rst b/src/rex.deploy/test/test_view.rst new file mode 100644 index 000000000..0eeab061d --- /dev/null +++ b/src/rex.deploy/test/test_view.rst @@ -0,0 +1,104 @@ +******************* + Deploying views +******************* + +.. contents:: Table of Contents + +Parsing view record +=================== + +We start with creating a test database and a ``Driver`` instance:: + + >>> from rex.deploy import Cluster + >>> cluster = Cluster('pgsql:deploy_demo_view') + >>> cluster.overwrite() + >>> driver = cluster.drive(logging=True) + +Field ``view`` denotes a table fact:: + + >>> fact = driver.parse("""{ view: one, definition: "select 1 as n" }""") + + >>> fact + ViewFact('one', definition='select 1 as n') + >>> print(fact) + view: one + definition: select 1 as n + +Creating the view +================= + +Deploying a view fact creates the view:: + + >>> driver("""{ view: one, definition: "select 1 as n" }""") + CREATE VIEW "one" AS (select 1 as n); + + >>> schema = driver.get_schema() + >>> 'one' in schema + True + +Deploying the same fact second time has no effect:: + + >>> driver("""{ view: one, definition: "select 1 as n" }""") + +Renaming the view +================= + +Deploying a view fact with ``was`` and another name will rename the view:: + + >>> driver("""{ view: one2, was: one}""") + ALTER VIEW "one" RENAME TO "one2"; + +Altering view definition +======================== + +Deploying a view fact with another definition will re-create the view:: + + >>> driver("""{ view: one2, definition: "select 2 as n" }""") + DROP VIEW "one2"; + CREATE VIEW "one2" AS (select 2 as n); + +Deploying the same fact second time has no effect:: + + >>> driver("""{ view: one2, definition: "select 2 as n" }""") + +Dropping the view +================= + +You can use ``ViewFact`` to remove a view:: + + >>> driver("""{ view: one2, present: false }""") + DROP VIEW "one2"; + +Deploying the same fact second time has no effect:: + + >>> driver("""{ view: one2, present: false }""") + +Views which depend on one another +================================= + +Let's create another view which uses the original ``one`` view in its +definition:: + + >>> driver(""" + ... - view: one + ... definition: select 1 as n + ... - view: one_plus + ... definition: select n + 1 from one + ... """) + CREATE VIEW "one" AS (select 1 as n); + CREATE VIEW "one_plus" AS (select n + 1 from one); + +Now we can try altering the ``one`` view which has dependents:: + + >>> driver("""{ view: one, definition: "select 2 as n" }""") # doctest: +ELLIPSIS + ... + Traceback (most recent call last): + ... + rex.core.Error: Got an error from the database driver: + cannot drop view one because other objects depend on it + DETAIL: view one_plus depends on view one + HINT: Use DROP ... CASCADE to drop the dependent objects too. + While executing SQL: + DROP VIEW "one"; + While deploying view fact: + "", line 1