Update SqlAlchemy to 1.4

Migrated spinedb_api to use SQLAlchemy 1.4.
- We now use vanilla SQLAlchemy queries, our custom query module
  is gone.
- DatabaseMapping's subqueries now return SQLAlchemy Result objects
  which is a BREAKING change.
- Some migrations towards SQLAlchemy 2.0 are included but we still
  get a lot of warnings with the SQLALCHEMY_WARN_20=1
  environment variable.
- Unit tests pass which is reassuring.

Re #121

CHANGELOG.md

@@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- **Breaking change:** The required SQLAlchemy version is now 1.4.
+  This may break clients that use the low-level query interface of `DatabaseMapping`,
+  e.g. `entity_sq`, as the queries now return proper SQLAlchemy 1.4 `Result` objects.
+  The high-level functions (`get_item`, `add_item` etc.) work as usual.
+
 ### Deprecated
 
 ### Removed

pyproject.toml

@@ -13,8 +13,7 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-    # v1.4 does not pass tests
-    "SQLAlchemy >=1.3, <1.4",
+    "SQLAlchemy >=1.4, <1.5",
     "alembic >=1.7",
     "datapackage >=1.15.2",
     "python-dateutil >=2.8.1",

spinedb_api/alembic/env.py

@@ -1,4 +1,3 @@
-from __future__ import with_statement
 from logging.config import fileConfig
 from alembic import context
 from sqlalchemy import engine_from_config, pool

spinedb_api/alembic/versions/02581198a2d8_create_entity_class_display_order_tables.py

@@ -32,20 +32,20 @@ def upgrade():
         sa.Column("display_order", sa.Integer, nullable=False),
         sa.Column(
             "display_status",
-            sa.Enum(DisplayStatus, name="display_status_enum"),
+            sa.Enum(DisplayStatus, name="display_status_enum", create_constraint=True),
             server_default=DisplayStatus.visible.name,
             nullable=False,
         ),
         sa.Column("display_font_color", sa.String(6), server_default=sa.null()),
         sa.Column("display_background_color", sa.String(6), server_default=sa.null()),
         sa.ForeignKeyConstraint(
-            ["entity_class_display_mode_id"],
+            ("entity_class_display_mode_id",),
             ["entity_class_display_mode.id"],
             onupdate="CASCADE",
             ondelete="CASCADE",
         ),
         sa.ForeignKeyConstraint(
-            ["entity_class_id"],
+            ("entity_class_id",),
             ["entity_class.id"],
             onupdate="CASCADE",
             ondelete="CASCADE",

spinedb_api/alembic/versions/0c7d199ae915_add_list_value_table.py

@@ -30,8 +30,8 @@ def upgrade():
     tfm = session.query(Base.classes.tool_feature_method).all()
     session.query(Base.classes.parameter_value_list).delete()
     session.query(Base.classes.tool_feature_method).delete()
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     # Change schema
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:

spinedb_api/alembic/versions/1892adebc00f_create_metadata_tables.py

@@ -17,8 +17,8 @@
 
 
 def upgrade():
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:
             batch_op.add_column(sa.Column("metadata_id", sa.Integer, server_default=sa.null()))

spinedb_api/alembic/versions/39e860a11b05_add_alternatives_and_scenarios.py

@@ -9,6 +9,7 @@
 from datetime import datetime, timezone
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import text
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import sessionmaker
 
@@ -90,8 +91,8 @@ def alter_tables_after_update():
             None, "alternative", ("alternative_id",), ("id",), onupdate="CASCADE", ondelete="CASCADE"
         )
 
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:
             batch_op.add_column(sa.Column("alternative_id", sa.Integer, server_default=sa.null()))
@@ -101,15 +102,17 @@ def alter_tables_after_update():
         date = datetime.utcnow()
         conn = op.get_bind()
         conn.execute(
-            """
+            text(
+                """
             UPDATE next_id
             SET
                 user = :user,
                 date = :date,
                 alternative_id = 2,
                 scenario_id = 1,
                 scenario_alternative_id = 1
-            """,
+            """
+            ),
             user=user,
             date=date,
         )

spinedb_api/alembic/versions/51fd7b69acf7_add_parameter_tag_and_parameter_value_list.py

@@ -17,8 +17,8 @@
 
 
 def upgrade():
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:
             batch_op.add_column(sa.Column("parameter_tag_id", sa.Integer))

spinedb_api/alembic/versions/6b7c994c1c61_drop_object_and_relationship_tables.py

@@ -8,6 +8,7 @@
 
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import text
 from spinedb_api.helpers import naming_convention
 
 # revision identifiers, used by Alembic.
@@ -106,28 +107,28 @@ def upgrade():
 
 def _get_constraints():
     conn = op.get_bind()
-    meta = sa.MetaData(conn)
-    meta.reflect()
+    meta = sa.MetaData()
+    meta.reflect(conn)
     return [[c.name for c in meta.tables[tname].constraints] for tname in ["entity_class", "entity"]]
 
 
 def _persist_data():
     conn = op.get_bind()
-    meta = sa.MetaData(conn)
-    meta.reflect()
+    meta = sa.MetaData()
+    meta.reflect(conn)
     ecd_items = [
-        {"entity_class_id": x["entity_class_id"], "dimension_id": x["member_class_id"], "position": x["dimension"]}
-        for x in conn.execute("SELECT * FROM relationship_entity_class")
+        {"entity_class_id": x.entity_class_id, "dimension_id": x.member_class_id, "position": x.dimension}
+        for x in conn.execute(text("SELECT * FROM relationship_entity_class"))
     ]
     ee_items = [
         {
-            "entity_id": x["entity_id"],
-            "entity_class_id": x["entity_class_id"],
-            "element_id": x["member_id"],
-            "dimension_id": x["member_class_id"],
-            "position": x["dimension"],
+            "entity_id": x.entity_id,
+            "entity_class_id": x.entity_class_id,
+            "element_id": x.member_id,
+            "dimension_id": x.member_class_id,
+            "position": x.dimension,
         }
-        for x in conn.execute("SELECT * FROM relationship_entity")
+        for x in conn.execute(text("SELECT * FROM relationship_entity"))
     ]
     op.bulk_insert(meta.tables["entity_class_dimension"], ecd_items)
     op.bulk_insert(meta.tables["entity_element"], ee_items)

spinedb_api/alembic/versions/7e2e66ae0f8f_rename_entity_class_display_mode_tables.py

@@ -8,6 +8,7 @@
 
 from alembic import op
 import sqlalchemy as sa
+from spinedb_api import naming_convention
 
 # revision identifiers, used by Alembic.
 revision = "7e2e66ae0f8f"
@@ -17,8 +18,8 @@
 
 
 def upgrade():
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     with op.batch_alter_table("entity_class_display_mode") as batch_op:
         batch_op.drop_constraint("pk_entity_class_display_mode", type_="primary")
         batch_op.create_primary_key("pk_display_mode", ["id"])

spinedb_api/alembic/versions/8c19c53d5701_rename_parameter_to_parameter_definition.py

@@ -22,8 +22,8 @@
 
 
 def upgrade():
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:
             batch_op.alter_column("parameter_id", new_column_name="parameter_definition_id", type_=sa.Integer)

spinedb_api/alembic/versions/989fccf80441_replace_values_with_reference_to_list_.py

@@ -25,8 +25,8 @@ def upgrade():
     conn = op.get_bind()
     Session = sessionmaker(bind=conn)
     session = Session()
-    meta = MetaData(conn)
-    meta.reflect()
+    meta = MetaData()
+    meta.reflect(conn)
     list_value = meta.tables["list_value"]
     parameter_definition = meta.tables["parameter_definition"]
     parameter_value = meta.tables["parameter_value"]

spinedb_api/alembic/versions/9da58d2def22_create_entity_group_table.py

@@ -17,8 +17,8 @@
 
 
 def upgrade():
-    m = sa.MetaData(op.get_bind())
-    m.reflect()
+    m = sa.MetaData()
+    m.reflect(op.get_bind())
     if "next_id" in m.tables:
         with op.batch_alter_table("next_id") as batch_op:
             batch_op.add_column(sa.Column("entity_group_id", sa.Integer))