Update id column of treenode_connector_edge and connector_geom table

This migration rewrites the tables treenode_connector_edge and
connector_geom, because their references to the treenode_connector table
are wrong and need to be updated. Up to this migration the id column was
of type int, but since it references the treenode_edge id column and the
connector id column, respectively, and they should follow their type,
which is bigint. This is updated by this migration by creating a new
table. This has the benefit that cleaning up the old table is easier and
doesn't need a VACUUM FULL run to reclaim space.

Both The treenode_connector_edge table and the connector_geom table
aren't tracked by the history system, which makes this update simpler.
In principle it could also be merged into the previous 64 Bit concept ID
migration, but it is easier for some existing test systems to not be
re-migrated.

The changes to the connector tables have been merged alrady separately
into the dev branch in commit 65b4044, but the migration didn't
allow to avoid VACUUM FULL to reclaim space, therefore the existing
version is updated accordingly. The existing version rewrote the table
directly, which would only allow to reclaim space with VACCUUM FULL.
Since this isn't very practical on large tables with many connectors,
this new version is introduced. The schema outcome is the same and
therefore no action has to be taken, if the previous version has been
applied already.

django/applications/catmaid/migrations/0088_connector_geom_fix_bigint.py

@@ -1,21 +1,99 @@
-# Generated by Django 2.2.5 on 2019-10-28 11:32
-
 from django.db import migrations
 
+
 forward = """
-	ALTER TABLE connector_geom ALTER COLUMN id TYPE bigint;
+    BEGIN;
+
+    ALTER TABLE connector_geom RENAME TO connector_geom_old;
+
+    -- Create new version of the connector_geom table which uses the bigint
+    -- data type for its ID. All other constraints will be added after the data
+    -- is copied.
+    CREATE TABLE connector_geom (
+        id  bigint PRIMARY KEY NOT NULL,
+        project_id integer NOT NULL,
+        geom geometry(PointZ) NOT NULL
+    );
+
+    INSERT INTO connector_geom (id, project_id, geom)
+    SELECT id, project_id, geom FROM connector_geom_old;
+
+    DROP TABLE connector_geom_old;
+
+    COMMIT;
 """
 
+
 backward = """
-	ALTER TABLE connector_geom ALTER COLUMN id TYPE integer;
+    BEGIN;
+
+    ALTER TABLE connector_geom RENAME TO connector_geom_old;
+
+    -- Create old version of the connector_geom table, which uses the bigint
+    -- data type for its ID. All other constraints will be added after the data
+    -- is copied.
+    CREATE TABLE connector_geom (
+        id  integer PRIMARY KEY NOT NULL,
+        project_id integer NOT NULL,
+        geom geometry(PointZ) NOT NULL
+    );
+
+    INSERT INTO connector_geom (id, project_id, geom)
+    SELECT id, project_id, geom FROM connector_geom_old;
+
+    DROP TABLE connector_geom_old;
+
+    COMMIT;
+"""
+
+create_indices_and_constraints = """
+    ALTER TABLE connector_geom
+        ADD CONSTRAINT connector_geom_id_fkey FOREIGN KEY (id)
+        REFERENCES connector (id) DEFERRABLE INITIALLY DEFERRED;
+    ALTER TABLE connector_geom
+        ADD CONSTRAINT connector_geom_project_id_fkey FOREIGN KEY (project_id)
+        REFERENCES project (id) DEFERRABLE INITIALLY DEFERRED;
+
+    CREATE INDEX connector_geom_2d_gist ON connector_geom
+        USING gist (geom);
+    CREATE INDEX connector_geom_gix ON connector_geom
+        USING gist (geom gist_geometry_ops_nd);
+    CREATE INDEX connector_geom_project_index ON connector_geom
+        USING btree (project_id);
+    CREATE INDEX connector_geom_z_range_gist ON connector_geom
+        USING gist (floatrange(st_zmin(geom::box3d), st_zmax(geom::box3d), '[]'::text));
+"""
+
+db_maintenance = """
+    VACUUM ANALYZE;
 """
 
+
 class Migration(migrations.Migration):
+    """This migration rewrites the table connector_geom. because its references
+    to the connector table are wrong and need to be updated. Up to this
+    migration the id column was of type int, but since it references the
+    treenode_edge id column (and the connector id column), it should follow its
+    type, which is bigint.  This is updated by this migration by creating a new
+    table. This has the benefit that cleaning up the old table is easier and
+    doesn't need a VACUUM FULL run.
+
+    This is a new version of the already existing version of this migration.
+    The existing version rewrote the table directly, which would only allow to
+    reclaim space with VACCUUM FULL. Since this isn't very practical on large
+    tables with many connectors, this new version is introduced. The schema
+    outcome is the same and therefore no action has to be taken, if the
+    previous version has been applied already.
+    """
 
     dependencies = [
         ('catmaid', '0087_add_nblast_normalization_mode'),
     ]
 
     operations = [
-            migrations.RunSQL(forward, backward)
+        migrations.RunSQL(migrations.RunSQL.noop, db_maintenance),
+        migrations.RunSQL(migrations.RunSQL.noop, create_indices_and_constraints),
+        migrations.RunSQL(forward, backward),
+        migrations.RunSQL(create_indices_and_constraints, migrations.RunSQL.noop),
+        migrations.RunSQL(db_maintenance, migrations.RunSQL.noop),
     ]

django/applications/catmaid/migrations/0089_fix_treenode_connector_edge_foreign_key_types.py

@@ -0,0 +1,92 @@
+from django.db import migrations
+
+
+forward = """
+    BEGIN;
+
+    ALTER TABLE treenode_connector_edge RENAME TO treenode_connector_edge_old;
+
+    -- Create new version of the treenode_connector_edge table, which uses the
+    -- bigint data type for its ID. All other constraints will be added after
+    -- the data is copied.
+    CREATE TABLE treenode_connector_edge (
+        id  bigint PRIMARY KEY NOT NULL,
+        project_id integer NOT NULL,
+        edge geometry(LinestringZ) NOT NULL
+    );
+
+    INSERT INTO treenode_connector_edge (id, project_id, edge)
+    SELECT id, project_id, edge FROM treenode_connector_edge_old;
+
+    DROP TABLE treenode_connector_edge_old;
+
+    COMMIT;
+"""
+
+
+backward = """
+    BEGIN;
+
+    ALTER TABLE treenode_connector_edge RENAME TO treenode_connector_edge_old;
+
+    -- Create old version of the treenode_connector_edge table which uses the
+    -- bigint data type for its ID. All other constraints will be added after
+    -- the data is copied.
+    CREATE TABLE treenode_connector_edge (
+        id  integer PRIMARY KEY NOT NULL,
+        project_id integer NOT NULL,
+        edge geometry(LinestringZ) NOT NULL
+    );
+
+    INSERT INTO treenode_connector_edge (id, project_id, edge)
+    SELECT id, project_id, edge FROM treenode_connector_edge_old;
+
+    DROP TABLE treenode_connector_edge_old;
+
+    COMMIT;
+"""
+
+create_indices_and_constraints = """
+    ALTER TABLE treenode_connector_edge
+        ADD CONSTRAINT treenode_connector_edge_id_fkey FOREIGN KEY (id)
+        REFERENCES treenode_connector_edge (id) DEFERRABLE INITIALLY DEFERRED;
+    ALTER TABLE treenode_connector_edge
+        ADD CONSTRAINT treenode_connector_edge_project_id_fkey FOREIGN KEY (project_id)
+        REFERENCES project (id) DEFERRABLE INITIALLY DEFERRED;
+
+    CREATE INDEX treenode_connector_edge_2d_gist ON treenode_connector_edge
+        USING gist (edge);
+    CREATE INDEX treenode_connector_edge_gix ON treenode_connector_edge
+        USING gist (edge gist_geometry_ops_nd);
+    CREATE INDEX treenode_connector_edge_project_index ON treenode_connector_edge
+        USING btree (project_id);
+    CREATE INDEX treenode_connector_edge_z_range_gist ON treenode_connector_edge
+        USING gist (floatrange(st_zmin(edge::box3d), st_zmax(edge::box3d), '[]'::text));
+"""
+
+db_maintenance = """
+    VACUUM ANALYZE;
+"""
+
+
+class Migration(migrations.Migration):
+    """This migration rewrites the table treenode_connector_edge, because their
+    references to the tables treenode_connector and connector are wrong and
+    need to be updated. Up to this migration the id column was of type int, but
+    since it references the treenode_edge id column , it should follow its
+    type, which is bigint.  This is updated by this migration by creating a new
+    table. This has the benefit that cleaning up the old table is easier and
+    doesn't need a VACUUM FULL run.
+    """
+
+    dependencies = [
+        ('catmaid', '0088_connector_geom_fix_bigint'),
+    ]
+
+    operations = [
+        migrations.RunSQL(migrations.RunSQL.noop, db_maintenance),
+        migrations.RunSQL(migrations.RunSQL.noop, create_indices_and_constraints),
+        migrations.RunSQL(forward, backward),
+        migrations.RunSQL(create_indices_and_constraints, migrations.RunSQL.noop),
+        migrations.RunSQL(db_maintenance, migrations.RunSQL.noop),
+    ]