diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 00000000..ec46a882 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,19 @@ + + + + + mariadb + true + org.mariadb.jdbc.Driver + jdbc:mariadb://old-db.development.svc.macrostrat.org:3306/macrostrat + $ProjectFileDir$ + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://db.development.svc.macrostrat.org:5432/macrostrat + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..00bf5117 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,69 @@ + + + + \ No newline at end of file diff --git a/cli/macrostrat/cli/_dev/dump_database.py b/cli/macrostrat/cli/_dev/dump_database.py index 2c1acdac..fc14ba99 100644 --- a/cli/macrostrat/cli/_dev/dump_database.py +++ b/cli/macrostrat/cli/_dev/dump_database.py @@ -6,7 +6,8 @@ from macrostrat.utils import get_logger from sqlalchemy.engine import Engine -from .utils import _create_command, print_stdout, print_stream_progress +from .utils import _create_command +from .stream_utils import print_stream_progress, print_stdout log = get_logger(__name__) diff --git a/cli/macrostrat/cli/_dev/restore_database.py b/cli/macrostrat/cli/_dev/restore_database.py index f4f535a5..feeef48f 100644 --- a/cli/macrostrat/cli/_dev/restore_database.py +++ b/cli/macrostrat/cli/_dev/restore_database.py @@ -10,9 +10,8 @@ from .utils import ( _create_command, _create_database_if_not_exists, - print_stdout, - print_stream_progress, ) +from .stream_utils import print_stream_progress, print_stdout console = Console() @@ -26,10 +25,8 @@ def pg_restore(*args, **kwargs): async def _pg_restore( engine: Engine, - *, + *args, create=False, - command_prefix: Optional[list] = None, - args: list = [], postgres_container: str = "postgres:15", ): # Pipe file to pg_restore, mimicking @@ -42,11 +39,10 @@ async def _pg_restore( # host, if possible, is probably the fastest option. There should be # multiple options ideally. _cmd = _create_command( - engine, "pg_restore", "-d", - args=args, - prefix=command_prefix, + engine, + *args, container=postgres_container, ) diff --git a/cli/macrostrat/cli/_dev/stream_utils.py b/cli/macrostrat/cli/_dev/stream_utils.py new file mode 100644 index 00000000..783a445d --- /dev/null +++ b/cli/macrostrat/cli/_dev/stream_utils.py @@ -0,0 +1,117 @@ +import asyncio +import sys +import zlib + +from aiofiles.threadpool import AsyncBufferedIOBase +from macrostrat.utils import get_logger +from .utils import console + +log = get_logger(__name__) + + +async def print_stream_progress( + input: asyncio.StreamReader | asyncio.subprocess.Process, + out_stream: asyncio.StreamWriter | None, + *, + verbose: bool = False, + chunk_size: int = 1024, + prefix: str = None, +): + """This should be unified with print_stream_progress, but there seem to be + slight API differences between aiofiles and asyncio.StreamWriter APIs.?""" + in_stream = input + if isinstance(in_stream, asyncio.subprocess.Process): + in_stream = input.stdout + + megabytes_written = 0 + i = 0 + + # Iterate over the stream by chunks + try: + while True: + chunk = await in_stream.read(chunk_size) + if not chunk: + log.info("End of stream") + break + if verbose: + log.info(chunk) + megabytes_written += len(chunk) / 1_000_000 + if isinstance(out_stream, AsyncBufferedIOBase): + await out_stream.write(chunk) + await out_stream.flush() + elif out_stream is not None: + out_stream.write(chunk) + await out_stream.drain() + i += 1 + if i == 100: + i = 0 + _print_progress(megabytes_written, end="\r", prefix=prefix) + except asyncio.CancelledError: + pass + finally: + _print_progress(megabytes_written, prefix=prefix) + + if isinstance(out_stream, AsyncBufferedIOBase): + out_stream.close() + elif out_stream is not None: + out_stream.close() + await out_stream.wait_closed() + + +def _print_progress(megabytes: float, **kwargs): + prefix = kwargs.pop("prefix", None) + if prefix is None: + prefix = "Dumped" + progress = f"{prefix} {megabytes:.1f} MB" + kwargs["file"] = sys.stderr + print(progress, **kwargs) + + +async def print_stdout(stream: asyncio.StreamReader): + async for line in stream: + log.info(line) + console.print(line.decode("utf-8"), style="dim") + + +class DecodingStreamReader(asyncio.StreamReader): + """A StreamReader that decompresses gzip files (if compressed)""" + + # https://ejosh.co/de/2022/08/stream-a-massive-gzipped-json-file-in-python/ + + def __init__(self, stream, encoding="utf-8", errors="strict"): + super().__init__() + self.stream = stream + self._is_gzipped = None + self.d = zlib.decompressobj(zlib.MAX_WBITS | 16) + + def decompress(self, input: bytes) -> bytes: + decompressed = self.d.decompress(input) + data = b"" + while self.d.unused_data != b"": + buf = self.d.unused_data + self.d = zlib.decompressobj(zlib.MAX_WBITS | 16) + data = self.d.decompress(buf) + return decompressed + data + + def transform_data(self, data): + if self._is_gzipped is None: + self._is_gzipped = data[:2] == b"\x1f\x8b" + log.info("is_gzipped: %s", self._is_gzipped) + if self._is_gzipped: + # Decompress the data + data = self.decompress(data) + return data + + async def read(self, n=-1): + data = await self.stream.read(n) + return self.transform_data(data) + + async def readline(self): + res = b"" + while res == b"": + # Read next line + line = await self.stream.readline() + if not line: + break + res += self.transform_data(line) + return res diff --git a/cli/macrostrat/cli/_dev/transfer_tables.py b/cli/macrostrat/cli/_dev/transfer_tables.py index 0a4084dd..02e084b8 100644 --- a/cli/macrostrat/cli/_dev/transfer_tables.py +++ b/cli/macrostrat/cli/_dev/transfer_tables.py @@ -1,5 +1,5 @@ import asyncio -from .utils import print_stream_progress, print_stdout +from .stream_utils import print_stream_progress, print_stdout from sqlalchemy.engine import Engine from .dump_database import _pg_dump from .restore_database import _pg_restore diff --git a/cli/macrostrat/cli/_dev/utils.py b/cli/macrostrat/cli/_dev/utils.py index 650b856b..13155e6d 100644 --- a/cli/macrostrat/cli/_dev/utils.py +++ b/cli/macrostrat/cli/_dev/utils.py @@ -1,13 +1,12 @@ -import asyncio from urllib.parse import quote import sys -from aiofiles.threadpool.binary import AsyncBufferedIOBase from macrostrat.utils import get_logger from rich.console import Console from sqlalchemy.engine import Engine from sqlalchemy.engine.url import URL -from sqlalchemy_utils import create_database, database_exists +from sqlalchemy_utils import create_database, database_exists, drop_database +from macrostrat.core.exc import MacrostratError console = Console() @@ -20,53 +19,64 @@ def _docker_local_run_args(postgres_container: str = "postgres:15"): "docker", "run", "-i", + "--attach", + "stdin", + "--attach", + "stdout", + "--attach", + "stderr", + "--log-driver", + "none", "--rm", - "--network", - "host", postgres_container, ] -def _create_database_if_not_exists(_url: URL, create=False): +def _create_database_if_not_exists( + _url: URL, *, create=False, allow_exists=True, overwrite=False +): database = _url.database + if overwrite: + create = True db_exists = database_exists(_url) if db_exists: - console.print(f"Database [bold cyan]{database}[/] already exists") + msg = f"Database [bold underline]{database}[/] already exists" + if overwrite: + console.print(f"{msg}, overwriting") + drop_database(_url) + db_exists = False + elif not allow_exists: + raise MacrostratError(msg, details="Use `--overwrite` to overwrite") + else: + console.print(msg) if create and not db_exists: console.print(f"Creating database [bold cyan]{database}[/]") create_database(_url) if not db_exists and not create: - raise ValueError( + raise MacrostratError( f"Database [bold cyan]{database}[/] does not exist. " "Use `--create` to create it." ) def _create_command( - engine: Engine, *command, - args=[], - prefix=None | list[str], - container="postgres:16", + container=None | str, ): - command_prefix = prefix or _docker_local_run_args(container) - _cmd = [*command_prefix, *command, str(engine.url), *args] - - log.info(" ".join(_cmd)) - - # Replace asterisks with the real password (if any). This is kind of backwards - # but it works. - if "***" in str(engine.url) and engine.url.password is not None: - _cmd = [ - *command_prefix, - *command, - raw_database_url(engine.url), - *args, - ] + """Create a command for operating on a database""" + _args = [] + if container is not None: + _args = _docker_local_run_args(container) - return _cmd + for arg in command: + if isinstance(arg, Engine): + arg = arg.url + if isinstance(arg, URL): + arg = raw_database_url(arg) + _args.append(arg) + return _args async def print_stream_progress( @@ -109,4 +119,8 @@ async def print_stdout(stream: asyncio.StreamReader): def raw_database_url(url: URL): - return str(url).replace("***", quote(url.password, safe="")) + """Replace the password asterisks with the actual password, in order to pass to other commands.""" + _url = str(url) + if "***" not in _url or url.password is None: + return _url + return _url.replace("***", quote(url.password, safe="")) diff --git a/cli/macrostrat/cli/cli.py b/cli/macrostrat/cli/cli.py index afb478c6..afdb9b53 100644 --- a/cli/macrostrat/cli/cli.py +++ b/cli/macrostrat/cli/cli.py @@ -270,6 +270,18 @@ def update_tileserver(db): app.subsystems.add(MacrostratAPISubsystem(app)) +# Mariadb CLI +if mariadb_url := getattr(settings, "mysql_database", None): + from .database.mariadb import app as mariadb_app + + main.add_typer( + mariadb_app, + name="mariadb", + rich_help_panel="Subsystems", + short_help="Manage the MariaDB database", + ) + + app.finish_loading_subsystems() diff --git a/cli/macrostrat/cli/commands/schlep.py b/cli/macrostrat/cli/commands/schlep.py deleted file mode 100644 index a448f05c..00000000 --- a/cli/macrostrat/cli/commands/schlep.py +++ /dev/null @@ -1,139 +0,0 @@ -from .base import Base -import sys -import datetime -from psycopg2.extensions import AsIs -from .table_meta import * -import urllib.request, urllib.error, urllib.parse - - -class Schlep(Base): - """ - macrostrat schlep : - Move a table from MariaDB to Postgres - - Available tables: - autocomplete - col_areas - refs - cols - intervals - timescales - liths - strat_names_meta - col_refs - concepts_places - units - lookup_strat_names - unit_strat_names - units_sections - unit_boundaries - strat_names - strat_tree - unit_liths - strat_names_places - col_groups - places - projects - lookup_unit_intervals - lith_atts - lookup_unit_liths - timescales_intervals - measures - measuremeta - measurements - - all - will move all above tables - - Usage: - macrostrat schlep
- macrostrat schlep all - macrostrat schlep -h | --help - Options: - -h --help Show this screen. - --version Show version. - Examples: - macrostrat schlep strat_names - Help: - For help using this tool, please open an issue on the Github repository: - https://github.com/UW-Macrostrat/macrostrat-cli - """ - - def move_table(self, table): - if table not in tables: - print("Table not found") - - print(" %s" % (table,)) - # Clean up - self.pg["cursor"].execute( - "DROP TABLE IF EXISTS macrostrat.%(table)s_new", {"table": AsIs(table)} - ) - self.pg["connection"].commit() - - # Create the new table in Postgres - self.pg["cursor"].execute(tables[table]["create"]) - self.pg["connection"].commit() - - # Dump the data from MariaDB - self.mariadb["cursor"].execute(tables[table]["dump"]) - - # Iterate on each row and insert into Postgres - row = self.mariadb["cursor"].fetchone() - while row is not None: - self.pg["cursor"].execute(tables[table]["insert"], row) - row = self.mariadb["cursor"].fetchone() - self.pg["connection"].commit() - - # Add any indexes - if "index" in tables[table] and len(tables[table]["index"].strip()) > 0: - self.pg["cursor"].execute(tables[table]["index"]) - self.pg["connection"].commit() - - # Run processing steps, if needed - if len(tables[table]["process"].strip()) != 0: - self.pg["cursor"].execute(tables[table]["process"]) - self.pg["connection"].commit() - - # Rename the table, drop the old one, add updated comment - self.pg["cursor"].execute( - """ - COMMENT ON TABLE macrostrat.%(table)s_new IS %(time)s; - ALTER TABLE IF EXISTS macrostrat.%(table)s RENAME TO %(table)s_old; - ALTER TABLE macrostrat.%(table)s_new RENAME TO %(table)s; - DROP TABLE IF EXISTS macrostrat.%(table)s_old CASCADE; - """, - { - "table": AsIs(table), - "time": "Last updated from MariaDB - " - + datetime.datetime.now().strftime("%Y-%m-%d %H:%M"), - }, - ) - self.pg["connection"].commit() - - def run(self): - print(self.args) - # Check if a command was provided - if len(self.args) == 1: - print("Please specify a table to move from MariaDB to Postgres") - for table in tables: - print(" %s" % (table,)) - sys.exit() - - # Validate the passed table - table = self.args[1] - if table not in tables and table != "all": - print("Invalid table") - sys.exit() - - if table == "all": - for t in tables: - Schlep.move_table(self, t) - else: - Schlep.move_table(self, table) - - try: - urllib.request.urlopen( - "http://127.0.0.1:5000/api/v2/columns/refresh-cache?cacheRefreshKey=%s" - % (self.credentials["cacheRefreshKey"],) - ).read() - except: - print("API cache was not updated") diff --git a/cli/macrostrat/cli/commands/table_meta.py b/cli/macrostrat/cli/commands/table_meta.py deleted file mode 100644 index 29678156..00000000 --- a/cli/macrostrat/cli/commands/table_meta.py +++ /dev/null @@ -1,30 +0,0 @@ -from collections import OrderedDict - -tables = OrderedDict({}) - -from os import path, listdir - -__here__ = path.dirname(__file__) -__table_meta__ = path.join(__here__, "table_meta") - -# Walk a directory tree and assemble a listing of SQL files -# Note: this is inefficient and should eventually be replaced with -# a function supporting the on-demand loading of SQL -for dirname in listdir(__table_meta__): - __dirpath__ = path.join(__table_meta__, dirname) - if not path.isdir(__dirpath__): - continue - table_name = dirname - if table_name not in tables: - tables[table_name] = OrderedDict({"process": ""}) - for fn in listdir(__dirpath__): - (base, ext) = path.splitext(fn) - # ix is a prefix that shows ordering, - (ix, operation) = base.split("-") - # Only accept SQL files (this way we can keep notes, etc.) - if ext != ".sql": - continue - fp = path.join(__dirpath__, fn) - with open(fp, "r") as f: - sqltext = f.read() - tables[table_name][operation] = sqltext diff --git a/cli/macrostrat/cli/commands/table_meta/autocomplete/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/autocomplete/0-dump.sql deleted file mode 100644 index 0b834892..00000000 --- a/cli/macrostrat/cli/commands/table_meta/autocomplete/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, name, type, category -FROM autocomplete - diff --git a/cli/macrostrat/cli/commands/table_meta/autocomplete/1-create.sql b/cli/macrostrat/cli/commands/table_meta/autocomplete/1-create.sql deleted file mode 100644 index 865f4096..00000000 --- a/cli/macrostrat/cli/commands/table_meta/autocomplete/1-create.sql +++ /dev/null @@ -1,8 +0,0 @@ - -CREATE TABLE macrostrat.autocomplete_new ( - id integer NOT NULL, - name text, - type text, - category text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/autocomplete/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/autocomplete/2-insert.sql deleted file mode 100644 index deb5863f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/autocomplete/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.autocomplete_new (id, name, type, category) VALUES (%(id)s, %(name)s, %(type)s, %(category)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/autocomplete/3-index.sql b/cli/macrostrat/cli/commands/table_meta/autocomplete/3-index.sql deleted file mode 100644 index 915b1a2b..00000000 --- a/cli/macrostrat/cli/commands/table_meta/autocomplete/3-index.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE INDEX ON macrostrat.autocomplete_new (id); -CREATE INDEX ON macrostrat.autocomplete_new (name); -CREATE INDEX ON macrostrat.autocomplete_new (type); -CREATE INDEX ON macrostrat.autocomplete_new (category); - diff --git a/cli/macrostrat/cli/commands/table_meta/col_groups/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/col_groups/0-dump.sql deleted file mode 100644 index e43aeef7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_groups/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, col_group, col_group_long -FROM col_groups - diff --git a/cli/macrostrat/cli/commands/table_meta/col_groups/1-create.sql b/cli/macrostrat/cli/commands/table_meta/col_groups/1-create.sql deleted file mode 100644 index ac779de4..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_groups/1-create.sql +++ /dev/null @@ -1,7 +0,0 @@ - -CREATE TABLE macrostrat.col_groups_new ( - id integer PRIMARY KEY, - col_group character varying(100), - col_group_long character varying(100) -); - diff --git a/cli/macrostrat/cli/commands/table_meta/col_groups/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/col_groups/2-insert.sql deleted file mode 100644 index b1e044cd..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_groups/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.col_groups_new (id, col_group, col_group_long ) VALUES (%(id)s, %(col_group)s, %(col_group_long)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/col_groups/3-index.sql b/cli/macrostrat/cli/commands/table_meta/col_groups/3-index.sql deleted file mode 100644 index 1582abe7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_groups/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ - -CREATE INDEX ON macrostrat.col_groups_new (id); - diff --git a/cli/macrostrat/cli/commands/table_meta/col_refs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/col_refs/0-dump.sql deleted file mode 100644 index 4afd34ca..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_refs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, col_id, ref_id -FROM col_refs - diff --git a/cli/macrostrat/cli/commands/table_meta/col_refs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/col_refs/1-create.sql deleted file mode 100644 index 43e44895..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_refs/1-create.sql +++ /dev/null @@ -1,7 +0,0 @@ - -CREATE TABLE macrostrat.col_refs_new ( - id integer PRIMARY KEY, - col_id integer, - ref_id integer -); - diff --git a/cli/macrostrat/cli/commands/table_meta/col_refs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/col_refs/2-insert.sql deleted file mode 100644 index b13a8652..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_refs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.col_refs_new (id, col_id, ref_id) VALUES (%(id)s, %(col_id)s, %(ref_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/col_refs/3-index.sql b/cli/macrostrat/cli/commands/table_meta/col_refs/3-index.sql deleted file mode 100644 index 4b256688..00000000 --- a/cli/macrostrat/cli/commands/table_meta/col_refs/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.col_refs_new (col_id); -CREATE INDEX ON macrostrat.col_refs_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/concepts_places/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/concepts_places/0-dump.sql deleted file mode 100644 index ca07b037..00000000 --- a/cli/macrostrat/cli/commands/table_meta/concepts_places/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT concept_id, place_id -FROM concepts_places - diff --git a/cli/macrostrat/cli/commands/table_meta/concepts_places/1-create.sql b/cli/macrostrat/cli/commands/table_meta/concepts_places/1-create.sql deleted file mode 100644 index 5ab44020..00000000 --- a/cli/macrostrat/cli/commands/table_meta/concepts_places/1-create.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE TABLE macrostrat.concepts_places_new ( - concept_id integer NOT NULL, - place_id integer NOT NULL -); - diff --git a/cli/macrostrat/cli/commands/table_meta/concepts_places/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/concepts_places/2-insert.sql deleted file mode 100644 index f4272af7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/concepts_places/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.concepts_places_new (concept_id, place_id) VALUES (%(concept_id)s, %(place_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/concepts_places/3-index.sql b/cli/macrostrat/cli/commands/table_meta/concepts_places/3-index.sql deleted file mode 100644 index 37166cf1..00000000 --- a/cli/macrostrat/cli/commands/table_meta/concepts_places/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.concepts_places_new (concept_id); -CREATE INDEX ON macrostrat.concepts_places_new (place_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/econs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/econs/0-dump.sql deleted file mode 100644 index b4b1adbd..00000000 --- a/cli/macrostrat/cli/commands/table_meta/econs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, econ, econ_type, econ_class, econ_color -FROM econs - diff --git a/cli/macrostrat/cli/commands/table_meta/econs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/econs/1-create.sql deleted file mode 100644 index 16051815..00000000 --- a/cli/macrostrat/cli/commands/table_meta/econs/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.econs_new ( - id integer NOT NULL PRIMARY KEY, - econ text, - econ_type text, - econ_class text, - econ_color text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/econs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/econs/2-insert.sql deleted file mode 100644 index 0d45e3d5..00000000 --- a/cli/macrostrat/cli/commands/table_meta/econs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.econs_new (id, econ, econ_type, econ_class, econ_color) VALUES (%(id)s, %(econ)s, %(econ_type)s, %(econ_class)s, %(econ_color)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/environs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/environs/0-dump.sql deleted file mode 100644 index f000ed95..00000000 --- a/cli/macrostrat/cli/commands/table_meta/environs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, environ, environ_type, environ_class, environ_color -FROM environs - diff --git a/cli/macrostrat/cli/commands/table_meta/environs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/environs/1-create.sql deleted file mode 100644 index b0517753..00000000 --- a/cli/macrostrat/cli/commands/table_meta/environs/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.environs_new ( - id integer NOT NULL PRIMARY KEY, - environ text, - environ_type text, - environ_class text, - environ_color text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/environs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/environs/2-insert.sql deleted file mode 100644 index cd8f95db..00000000 --- a/cli/macrostrat/cli/commands/table_meta/environs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.environs_new (id, environ, environ_type, environ_class, environ_color) VALUES (%(id)s, %(environ)s, %(environ_type)s, %(environ_class)s, %(environ_color)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/lith_atts/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/lith_atts/0-dump.sql deleted file mode 100644 index 1961ae66..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lith_atts/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, lith_att, att_type, lith_att_fill -FROM lith_atts - diff --git a/cli/macrostrat/cli/commands/table_meta/lith_atts/1-create.sql b/cli/macrostrat/cli/commands/table_meta/lith_atts/1-create.sql deleted file mode 100644 index df10ccc6..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lith_atts/1-create.sql +++ /dev/null @@ -1,8 +0,0 @@ - -CREATE TABLE macrostrat.lith_atts_new ( - id integer PRIMARY KEY NOT NULL, - lith_att character varying(75), - att_type character varying(25), - lith_att_fill integer -); - diff --git a/cli/macrostrat/cli/commands/table_meta/lith_atts/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/lith_atts/2-insert.sql deleted file mode 100644 index 9506dcf2..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lith_atts/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.lith_atts_new (id, lith_att, att_type, lith_att_fill) VALUES (%(id)s, %(lith_att)s, %(att_type)s, %(lith_att_fill)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/lith_atts/3-index.sql b/cli/macrostrat/cli/commands/table_meta/lith_atts/3-index.sql deleted file mode 100644 index a7e6f696..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lith_atts/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.lith_atts_new (att_type); -CREATE INDEX ON macrostrat.lith_atts_new (lith_att); - diff --git a/cli/macrostrat/cli/commands/table_meta/liths/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/liths/0-dump.sql deleted file mode 100644 index 13b4454a..00000000 --- a/cli/macrostrat/cli/commands/table_meta/liths/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, lith, lith_group, lith_type, lith_class, lith_equiv, lith_fill, comp_coef, initial_porosity, bulk_density, lith_color -FROM liths - diff --git a/cli/macrostrat/cli/commands/table_meta/liths/1-create.sql b/cli/macrostrat/cli/commands/table_meta/liths/1-create.sql deleted file mode 100644 index b46cde79..00000000 --- a/cli/macrostrat/cli/commands/table_meta/liths/1-create.sql +++ /dev/null @@ -1,15 +0,0 @@ - -CREATE TABLE macrostrat.liths_new ( - id integer PRIMARY KEY NOT NULL, - lith character varying(75), - lith_group text, - lith_type character varying(50), - lith_class character varying(50), - lith_equiv integer, - lith_fill integer, - comp_coef numeric, - initial_porosity numeric, - bulk_density numeric, - lith_color character varying(12) -); - diff --git a/cli/macrostrat/cli/commands/table_meta/liths/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/liths/2-insert.sql deleted file mode 100644 index 12684f72..00000000 --- a/cli/macrostrat/cli/commands/table_meta/liths/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.liths_new ( id, lith, lith_group, lith_type, lith_class, lith_equiv, lith_fill, comp_coef, initial_porosity, bulk_density, lith_color) VALUES (%(id)s, %(lith)s, %(lith_group)s, %(lith_type)s, %(lith_class)s, %(lith_equiv)s, %(lith_fill)s, %(comp_coef)s, %(initial_porosity)s, %(bulk_density)s, %(lith_color)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/liths/3-index.sql b/cli/macrostrat/cli/commands/table_meta/liths/3-index.sql deleted file mode 100644 index ba8835b3..00000000 --- a/cli/macrostrat/cli/commands/table_meta/liths/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.liths_new (lith); -CREATE INDEX ON macrostrat.liths_new (lith_class); -CREATE INDEX ON macrostrat.liths_new (lith_type); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/0-dump.sql deleted file mode 100644 index 73e11620..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT strat_name_id, strat_name, rank, concept_id, rank_name, bed_id, bed_name, mbr_id, mbr_name, fm_id, fm_name, gp_id, gp_name, sgp_id, sgp_name, early_age, late_age, gsc_lexicon, b_period, t_period, c_interval, name_no_lith -FROM lookup_strat_names - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/1-create.sql b/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/1-create.sql deleted file mode 100644 index a28b247d..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/1-create.sql +++ /dev/null @@ -1,26 +0,0 @@ - -CREATE TABLE macrostrat.lookup_strat_names_new ( - strat_name_id integer, - strat_name character varying(100), - rank character varying(20), - concept_id integer, - rank_name character varying(200), - bed_id integer, - bed_name character varying(100), - mbr_id integer, - mbr_name character varying(100), - fm_id integer, - fm_name character varying(100), - gp_id integer, - gp_name character varying(100), - sgp_id integer, - sgp_name character varying(100), - early_age numeric, - late_age numeric, - gsc_lexicon character varying(20), - b_period character varying(100), - t_period character varying(100), - c_interval character varying(100), - name_no_lith character varying(100) -); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/2-insert.sql deleted file mode 100644 index 57082065..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.lookup_strat_names_new (strat_name_id, strat_name, rank, concept_id, rank_name, bed_id, bed_name, mbr_id, mbr_name, fm_id, fm_name, gp_id, gp_name, sgp_id, sgp_name, early_age, late_age, gsc_lexicon, b_period, t_period, c_interval, name_no_lith) VALUES (%(strat_name_id)s, %(strat_name)s, %(rank)s, %(concept_id)s, %(rank_name)s, %(bed_id)s, %(bed_name)s, %(mbr_id)s, %(mbr_name)s, %(fm_id)s, %(fm_name)s, %(gp_id)s, %(gp_name)s, %(sgp_id)s, %(sgp_name)s, %(early_age)s, %(late_age)s, %(gsc_lexicon)s, %(b_period)s, %(t_period)s, %(c_interval)s, %(name_no_lith)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/3-index.sql b/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/3-index.sql deleted file mode 100644 index 40b0a376..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_strat_names/3-index.sql +++ /dev/null @@ -1,10 +0,0 @@ - -CREATE INDEX ON macrostrat.lookup_strat_names_new (strat_name_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (concept_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (bed_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (mbr_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (fm_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (gp_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (sgp_id); -CREATE INDEX ON macrostrat.lookup_strat_names_new (strat_name); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/0-dump.sql deleted file mode 100644 index d0cf75b1..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT unit_id, lith, environ, econ, measure_short, measure_long -FROM lookup_unit_attrs_api - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/1-create.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/1-create.sql deleted file mode 100644 index ebd97728..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/1-create.sql +++ /dev/null @@ -1,10 +0,0 @@ - -CREATE TABLE macrostrat.lookup_unit_attrs_api_new ( - unit_id integer, - lith json, - environ json, - econ json, - measure_short json, - measure_long json -); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/2-insert.sql deleted file mode 100644 index 48e9c771..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/2-insert.sql +++ /dev/null @@ -1,4 +0,0 @@ - -INSERT INTO macrostrat.lookup_unit_attrs_api_new (unit_id, lith, environ, econ, measure_short, measure_long) VALUES -(%(unit_id)s, encode(%(lith)s, 'escape')::json, encode(%(environ)s, 'escape')::json, encode(%(econ)s, 'escape')::json, encode(%(measure_short)s, 'escape')::json, encode(%(measure_long)s, 'escape')::json) - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/3-index.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/3-index.sql deleted file mode 100644 index 454a458b..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_attrs_api/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ - -CREATE INDEX ON macrostrat.lookup_unit_attrs_api_new (unit_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/0-dump.sql deleted file mode 100644 index f69c220c..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT unit_id, lith_class, lith_type, lith_short, lith_long, environ_class, environ_type, environ -FROM lookup_unit_liths - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/1-create.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/1-create.sql deleted file mode 100644 index ca8c770d..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/1-create.sql +++ /dev/null @@ -1,12 +0,0 @@ - -CREATE TABLE macrostrat.lookup_unit_liths_new ( - unit_id integer, - lith_class character varying(100), - lith_type character varying(100), - lith_short text, - lith_long text, - environ_class character varying(100), - environ_type character varying(100), - environ character varying(255) -); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/2-insert.sql deleted file mode 100644 index 47bcc6ba..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.lookup_unit_liths_new (unit_id, lith_class, lith_type, lith_short, lith_long, environ_class, environ_type, environ) VALUES (%(unit_id)s, %(lith_class)s, %(lith_type)s, %(lith_short)s, %(lith_long)s, %(environ_class)s, %(environ_type)s, %(environ)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/3-index.sql b/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/3-index.sql deleted file mode 100644 index 5b095501..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_unit_liths/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ - -CREATE INDEX ON macrostrat.lookup_unit_liths_new (unit_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_units/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/lookup_units/0-dump.sql deleted file mode 100644 index 3dd49e9a..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_units/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT unit_id, col_area, project_id, t_int, t_int_name, t_int_age, t_age, t_prop, t_plat, t_plng, b_int, b_int_name, b_int_age, b_age, b_prop, b_plat, b_plng, clat, clng, color, text_color, units_above, units_below, pbdb_collections, pbdb_occurrences, age, age_id, epoch, epoch_id, period, period_id, era, era_id, eon, eon_id -FROM lookup_units - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_units/1-create.sql b/cli/macrostrat/cli/commands/table_meta/lookup_units/1-create.sql deleted file mode 100644 index 2604013c..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_units/1-create.sql +++ /dev/null @@ -1,39 +0,0 @@ - -CREATE TABLE macrostrat.lookup_units_new ( - unit_id integer PRIMARY KEY NOT NULL, - col_area numeric NOT NULL, - project_id integer NOT NULL, - t_int integer, - t_int_name text, - t_int_age numeric, - t_age numeric, - t_prop numeric, - t_plat numeric, - t_plng numeric, - b_int integer, - b_int_name text, - b_int_age numeric, - b_age numeric, - b_prop numeric, - b_plat numeric, - b_plng numeric, - clat numeric, - clng numeric, - color text, - text_color text, - units_above text, - units_below text, - pbdb_collections integer, - pbdb_occurrences integer, - age text, - age_id integer, - epoch text, - epoch_id integer, - period text, - period_id integer, - era text, - era_id integer, - eon text, - eon_id integer -); - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_units/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/lookup_units/2-insert.sql deleted file mode 100644 index 14197062..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_units/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.lookup_units_new (unit_id, col_area, project_id, t_int, t_int_name, t_int_age, t_age, t_prop, t_plat, t_plng, b_int, b_int_name, b_int_age, b_age, b_prop, b_plat, b_plng, clat, clng, color, text_color, units_above, units_below, pbdb_collections, pbdb_occurrences, age, age_id, epoch, epoch_id, period, period_id, era, era_id, eon, eon_id) VALUES (%(unit_id)s, %(col_area)s, %(project_id)s, %(t_int)s, %(t_int_name)s, %(t_int_age)s, %(t_age)s, %(t_prop)s, %(t_plat)s, %(t_plng)s, %(b_int)s, %(b_int_name)s, %(b_int_age)s, %(b_age)s, %(b_prop)s, %(b_plat)s, %(b_plng)s, %(clat)s, %(clng)s, %(color)s, %(text_color)s, %(units_above)s, %(units_below)s, %(pbdb_collections)s, %(pbdb_occurrences)s, %(age)s, %(age_id)s, %(epoch)s, %(epoch_id)s, %(period)s, %(period_id)s, %(era)s, %(era_id)s, %(eon)s, %(eon_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/lookup_units/3-index.sql b/cli/macrostrat/cli/commands/table_meta/lookup_units/3-index.sql deleted file mode 100644 index 9dc7100a..00000000 --- a/cli/macrostrat/cli/commands/table_meta/lookup_units/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.lookup_units_new (project_id); -CREATE INDEX ON macrostrat.lookup_units_new (t_int); -CREATE INDEX ON macrostrat.lookup_units_new (b_int); - diff --git a/cli/macrostrat/cli/commands/table_meta/measurements/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/measurements/0-dump.sql deleted file mode 100644 index 48fb838f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measurements/0-dump.sql +++ /dev/null @@ -1,7 +0,0 @@ --- This one is executed on MariaDB -SELECT - id, - measurement_class, - measurement_type, - measurement -FROM measurements diff --git a/cli/macrostrat/cli/commands/table_meta/measurements/1-create.sql b/cli/macrostrat/cli/commands/table_meta/measurements/1-create.sql deleted file mode 100644 index a2c4c42d..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measurements/1-create.sql +++ /dev/null @@ -1,16 +0,0 @@ -DROP TYPE IF EXISTS measurement_class CASCADE; -DROP TYPE IF EXISTS measurement_type CASCADE; - -CREATE TYPE measurement_class AS ENUM( - '','geophysical','geochemical','sedimentological'); -CREATE TYPE measurement_type AS ENUM( - '','material properties','geochronological','major elements','minor elements', - 'radiogenic isotopes','stable isotopes','petrologic','environmental'); -COMMIT; - -CREATE TABLE macrostrat.measurements_new ( - id serial PRIMARY KEY NOT NULL, - measurement_class measurement_class NOT NULL, - measurement_type measurement_type NOT NULL, - measurement text NOT NULL -); diff --git a/cli/macrostrat/cli/commands/table_meta/measurements/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/measurements/2-insert.sql deleted file mode 100644 index 4d3180c7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measurements/2-insert.sql +++ /dev/null @@ -1,9 +0,0 @@ -INSERT INTO macrostrat.measurements_new ( - id, measurement_class, measurement_type, measurement) -VALUES ( - %(id)s, - %(measurement_class)s, - %(measurement_type)s, - %(measurement)s -) - diff --git a/cli/macrostrat/cli/commands/table_meta/measurements/3-index.sql b/cli/macrostrat/cli/commands/table_meta/measurements/3-index.sql deleted file mode 100644 index a1fa36ec..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measurements/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE INDEX ON macrostrat.measurements_new (id); -CREATE INDEX ON macrostrat.measurements_new (measurement_class); -CREATE INDEX ON macrostrat.measurements_new (measurement_type); - diff --git a/cli/macrostrat/cli/commands/table_meta/measurements/4-process.sql b/cli/macrostrat/cli/commands/table_meta/measurements/4-process.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/macrostrat/cli/commands/table_meta/measures/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/measures/0-dump.sql deleted file mode 100644 index 0de59ef1..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measures/0-dump.sql +++ /dev/null @@ -1,16 +0,0 @@ --- This one is executed on MariaDB -SELECT - id, - measuremeta_id, - measurement_id, - -- We had a problem with NUL characters in this column - replace(sample_no, CHAR(0x00 using utf8), "") sample_no, - measure_phase, - method, - units, - measure_value, - v_error, - v_error_units, - v_type, - v_n -FROM measures; diff --git a/cli/macrostrat/cli/commands/table_meta/measures/1-create.sql b/cli/macrostrat/cli/commands/table_meta/measures/1-create.sql deleted file mode 100644 index 99696ea8..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measures/1-create.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE macrostrat.measures_new ( - id serial, - measuremeta_id integer NOT NULL, -- REFERENCES macrostrat.measuremeta(id), - measurement_id integer NOT NULL, -- REFERENCES macrostrat.measurements(id), - sample_no varchar(50), - measure_phase varchar(100) NOT NULL, - method varchar(100) NOT NULL, - units varchar(25) NOT NULL, - measure_value decimal(10,5), - v_error decimal(10,5), - v_error_units varchar(25), - v_type varchar(100), - v_n integer -) diff --git a/cli/macrostrat/cli/commands/table_meta/measures/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/measures/2-insert.sql deleted file mode 100644 index 82921818..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measures/2-insert.sql +++ /dev/null @@ -1,29 +0,0 @@ -INSERT INTO macrostrat.measures_new ( - id, - measuremeta_id, - measurement_id, - sample_no, - measure_phase, - method, - units, - measure_value, - v_error, - v_error_units, - v_type, - v_n -) -VALUES ( - %(id)s, - %(measuremeta_id)s, - %(measurement_id)s, - %(sample_no)s, - %(measure_phase)s, - %(method)s, - %(units)s, - %(measure_value)s, - %(v_error)s, - %(v_error_units)s, - %(v_type)s, - %(v_n)s -) - diff --git a/cli/macrostrat/cli/commands/table_meta/measures/3-index.sql b/cli/macrostrat/cli/commands/table_meta/measures/3-index.sql deleted file mode 100644 index 7b6a03a6..00000000 --- a/cli/macrostrat/cli/commands/table_meta/measures/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX ON macrostrat.measures_new (measurement_id); -CREATE INDEX ON macrostrat.measures_new (measuremeta_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/places/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/places/0-dump.sql deleted file mode 100644 index eb376e5e..00000000 --- a/cli/macrostrat/cli/commands/table_meta/places/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT place_id, name, abbrev, postal, country, country_abbrev, ST_AsText(geom) geom -FROM places - diff --git a/cli/macrostrat/cli/commands/table_meta/places/1-create.sql b/cli/macrostrat/cli/commands/table_meta/places/1-create.sql deleted file mode 100644 index 6a3bf77c..00000000 --- a/cli/macrostrat/cli/commands/table_meta/places/1-create.sql +++ /dev/null @@ -1,11 +0,0 @@ - -CREATE TABLE macrostrat.places_new ( - place_id integer PRIMARY KEY, - name text, - abbrev text, - postal text, - country text, - country_abbrev text, - geom geometry -); - diff --git a/cli/macrostrat/cli/commands/table_meta/places/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/places/2-insert.sql deleted file mode 100644 index c8fe4028..00000000 --- a/cli/macrostrat/cli/commands/table_meta/places/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.places_new (place_id, name, abbrev, postal, country, country_abbrev, geom) VALUES (%(place_id)s, %(name)s, %(abbrev)s, %(postal)s, %(country)s, %(country_abbrev)s, ST_SetSRID(ST_GeomFromText(%(geom)s), 4326)) - diff --git a/cli/macrostrat/cli/commands/table_meta/places/3-index.sql b/cli/macrostrat/cli/commands/table_meta/places/3-index.sql deleted file mode 100644 index ac967ada..00000000 --- a/cli/macrostrat/cli/commands/table_meta/places/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ - -CREATE INDEX ON macrostrat.places_new USING GiST (geom); - diff --git a/cli/macrostrat/cli/commands/table_meta/projects/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/projects/0-dump.sql deleted file mode 100644 index c1187a82..00000000 --- a/cli/macrostrat/cli/commands/table_meta/projects/0-dump.sql +++ /dev/null @@ -1,6 +0,0 @@ -SELECT -id, -project, -descrip, -timescale_id -FROM projects \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/projects/1-create.sql b/cli/macrostrat/cli/commands/table_meta/projects/1-create.sql deleted file mode 100644 index 4e09cfe7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/projects/1-create.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE macrostrat.projects_new ( - id serial PRIMARY KEY, - project text, - descrip text, - timescale_id int -); \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/projects/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/projects/2-insert.sql deleted file mode 100644 index 8e6348db..00000000 --- a/cli/macrostrat/cli/commands/table_meta/projects/2-insert.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO macrostrat.projects_new (id, project, descrip, timescale_id) VALUES - (%(id)s, %(project)s, %(descrip)s, %(timescale_id)s); \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/projects/3-index.sql b/cli/macrostrat/cli/commands/table_meta/projects/3-index.sql deleted file mode 100644 index f7724bc2..00000000 --- a/cli/macrostrat/cli/commands/table_meta/projects/3-index.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX ON macrostrat.projects_new (project); -CREATE INDEX ON macrostrat.projects_new (timescale_id); \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/refs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/refs/0-dump.sql deleted file mode 100644 index a44c5d77..00000000 --- a/cli/macrostrat/cli/commands/table_meta/refs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, pub_year, author, ref, doi, compilation_code, url, ST_AsText(rgeom) rgeom -FROM refs - diff --git a/cli/macrostrat/cli/commands/table_meta/refs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/refs/1-create.sql deleted file mode 100644 index dc81db82..00000000 --- a/cli/macrostrat/cli/commands/table_meta/refs/1-create.sql +++ /dev/null @@ -1,12 +0,0 @@ - -CREATE TABLE macrostrat.refs_new ( - id integer PRIMARY key, - pub_year integer, - author character varying(255), - ref text, - doi character varying(40), - compilation_code character varying(100), - url text, - rgeom geometry -); - diff --git a/cli/macrostrat/cli/commands/table_meta/refs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/refs/2-insert.sql deleted file mode 100644 index bcada6b7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/refs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.refs_new (id, pub_year, author, ref, doi, compilation_code, url, rgeom) VALUES (%(id)s, %(pub_year)s, %(author)s, %(ref)s, %(doi)s, %(compilation_code)s, %(url)s, ST_SetSRID(ST_GeomFromText(%(rgeom)s), 4326)) - diff --git a/cli/macrostrat/cli/commands/table_meta/refs/3-index.sql b/cli/macrostrat/cli/commands/table_meta/refs/3-index.sql deleted file mode 100644 index e20b7c7f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/refs/3-index.sql +++ /dev/null @@ -1,3 +0,0 @@ - -CREATE INDEX ON macrostrat.refs_new USING GiST (rgeom); - diff --git a/cli/macrostrat/cli/commands/table_meta/sections/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/sections/0-dump.sql deleted file mode 100644 index c428c80f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/sections/0-dump.sql +++ /dev/null @@ -1 +0,0 @@ -select id, col_id from sections \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/sections/1-create.sql b/cli/macrostrat/cli/commands/table_meta/sections/1-create.sql deleted file mode 100644 index fa0f5f50..00000000 --- a/cli/macrostrat/cli/commands/table_meta/sections/1-create.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE macrostrat.sections_new( - id serial PRIMARY KEY, - col_id int -) \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/sections/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/sections/2-insert.sql deleted file mode 100644 index 9c82660f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/sections/2-insert.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO macrostrat.sections_new(id, col_id) VALUES (%(id)s,%(col_id)s) \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/sections/3-index.sql b/cli/macrostrat/cli/commands/table_meta/sections/3-index.sql deleted file mode 100644 index 7993dfcb..00000000 --- a/cli/macrostrat/cli/commands/table_meta/sections/3-index.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX ON macrostrat.sections_new(id); -CREATE INDEX ON macrostrat.sections_new(col_id); \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/strat_names/0-dump.sql deleted file mode 100644 index e73ee58f..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, strat_name, rank, ref_id, concept_id -FROM strat_names - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names/1-create.sql b/cli/macrostrat/cli/commands/table_meta/strat_names/1-create.sql deleted file mode 100644 index ff21c625..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.strat_names_new ( - id serial PRIMARY KEY NOT NULL, - strat_name character varying(100) NOT NULL, - rank character varying(50), - ref_id integer NOT NULL, - concept_id integer -) - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/strat_names/2-insert.sql deleted file mode 100644 index ffea9a22..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.strat_names_new (id, strat_name, rank, ref_id, concept_id) VALUES (%(id)s, %(strat_name)s, %(rank)s, %(ref_id)s, %(concept_id)s); - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names/3-index.sql b/cli/macrostrat/cli/commands/table_meta/strat_names/3-index.sql deleted file mode 100644 index 417de63e..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names/3-index.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE INDEX ON macrostrat.strat_names_new (strat_name); -CREATE INDEX ON macrostrat.strat_names_new (rank); -CREATE INDEX ON macrostrat.strat_names_new (ref_id); -CREATE INDEX ON macrostrat.strat_names_new (concept_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_meta/0-dump.sql deleted file mode 100644 index 6a4f6438..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT concept_id, orig_id, name, geologic_age, interval_id, b_int, t_int, usage_notes, other, province, url, ref_id -FROM strat_names_meta - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/1-create.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_meta/1-create.sql deleted file mode 100644 index 6d385af0..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/1-create.sql +++ /dev/null @@ -1,16 +0,0 @@ - -CREATE TABLE macrostrat.strat_names_meta_new ( - concept_id integer PRIMARY KEY, - orig_id integer NOT NULL, - name character varying(40), - geologic_age text, - interval_id integer NOT NULL, - b_int integer NOT NULL, - t_int integer NOT NULL, - usage_notes text, - other text, - province text, - url character varying(150), - ref_id integer NOT NULL -); - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_meta/2-insert.sql deleted file mode 100644 index 988ba9a4..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.strat_names_meta_new (concept_id, orig_id, name, geologic_age, interval_id, b_int, t_int, usage_notes, other, province, url, ref_id) VALUES (%(concept_id)s, %(orig_id)s, %(name)s, %(geologic_age)s, %(interval_id)s, %(b_int)s, %(t_int)s, %(usage_notes)s, %(other)s, %(province)s, %(url)s, %(ref_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/3-index.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_meta/3-index.sql deleted file mode 100644 index 99689c85..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_meta/3-index.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE INDEX ON macrostrat.strat_names_meta_new (interval_id); -CREATE INDEX ON macrostrat.strat_names_meta_new (b_int); -CREATE INDEX ON macrostrat.strat_names_meta_new (t_int); -CREATE INDEX ON macrostrat.strat_names_meta_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_places/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_places/0-dump.sql deleted file mode 100644 index ae301876..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_places/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT strat_name_id, place_id -FROM strat_names_places - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_places/1-create.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_places/1-create.sql deleted file mode 100644 index a0ea9b95..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_places/1-create.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE TABLE macrostrat.strat_names_places_new ( - strat_name_id integer NOT NULL, - place_id integer NOT NULL -); - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_places/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_places/2-insert.sql deleted file mode 100644 index 6989d450..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_places/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.strat_names_places_new (strat_name_id, place_id) VALUES (%(strat_name_id)s, %(place_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/strat_names_places/3-index.sql b/cli/macrostrat/cli/commands/table_meta/strat_names_places/3-index.sql deleted file mode 100644 index 6d76f3cc..00000000 --- a/cli/macrostrat/cli/commands/table_meta/strat_names_places/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.strat_names_places_new (strat_name_id); -CREATE INDEX ON macrostrat.strat_names_places_new (place_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/timescales/0-dump.sql deleted file mode 100644 index 01c1c7ec..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, timescale, ref_id -FROM timescales - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales/1-create.sql b/cli/macrostrat/cli/commands/table_meta/timescales/1-create.sql deleted file mode 100644 index c4c6ebb3..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales/1-create.sql +++ /dev/null @@ -1,7 +0,0 @@ - -CREATE TABLE macrostrat.timescales_new ( - id integer PRIMARY KEY, - timescale character varying(100), - ref_id integer -); - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/timescales/2-insert.sql deleted file mode 100644 index 00f12b4a..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.timescales_new (id, timescale, ref_id) VALUES (%(id)s, %(timescale)s, %(ref_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales/3-index.sql b/cli/macrostrat/cli/commands/table_meta/timescales/3-index.sql deleted file mode 100644 index 693a7ba1..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.timescales_new (timescale); -CREATE INDEX ON macrostrat.timescales_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/timescales_intervals/0-dump.sql deleted file mode 100644 index 4fe044c5..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT timescale_id, interval_id -FROM timescales_intervals - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/1-create.sql b/cli/macrostrat/cli/commands/table_meta/timescales_intervals/1-create.sql deleted file mode 100644 index 7cb2b69a..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/1-create.sql +++ /dev/null @@ -1,6 +0,0 @@ - -CREATE TABLE macrostrat.timescales_intervals_new ( - timescale_id integer, - interval_id integer -); - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/timescales_intervals/2-insert.sql deleted file mode 100644 index 4bed69a3..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.timescales_intervals_new (timescale_id, interval_id) VALUES (%(timescale_id)s, %(interval_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/3-index.sql b/cli/macrostrat/cli/commands/table_meta/timescales_intervals/3-index.sql deleted file mode 100644 index 502f1553..00000000 --- a/cli/macrostrat/cli/commands/table_meta/timescales_intervals/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.timescales_intervals_new (timescale_id); -CREATE INDEX ON macrostrat.timescales_intervals_new (interval_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_boundaries/0-dump.sql deleted file mode 100644 index 1c96b4b9..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/0-dump.sql +++ /dev/null @@ -1,15 +0,0 @@ -SELECT - id, - t1, - t1_prop, - t1_age, - unit_id, - unit_id_2, - section_id, - boundary_position, - boundary_type, - boundary_status, - paleo_lat, - paleo_lng, - ref_id -FROM unit_boundaries; \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_boundaries/1-create.sql deleted file mode 100644 index 35d1572e..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/1-create.sql +++ /dev/null @@ -1,21 +0,0 @@ -DROP TYPE IF EXISTS macrostrat.boundary_type CASCADE; -CREATE TYPE macrostrat.boundary_type AS ENUM('','unconformity','conformity','fault','disconformity','non-conformity','angular unconformity'); - -DROP TYPE IF EXISTS macrostrat.boundary_status CASCADE; -CREATE TYPE macrostrat.boundary_status AS ENUM('','modeled','relative','absolute','spike'); - -CREATE TABLE macrostrat.unit_boundaries ( - id serial PRIMARY KEY, - t1 numeric NOT NULL, - t1_prop decimal(6,5) NOT NULL, - t1_age decimal(8,4) NOT NULL, - unit_id integer NOT NULL, - unit_id_2 integer NOT NULL, - section_id integer NOT NULL, - boundary_position decimal(6,2) DEFAULT NULL, - boundary_type macrostrat.boundary_type NOT NULL DEFAULT '', - boundary_status macrostrat.boundary_status NOT NULL DEFAULT 'modeled', - paleo_lat decimal(8,5), - paleo_lng decimal(8,5), - ref_id integer NOT NULL DEFAULT 217 -); diff --git a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_boundaries/2-insert.sql deleted file mode 100644 index 2d6a3876..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/2-insert.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO macrostrat.unit_boundaries(id, t1, t1_prop, t1_age, unit_id, unit_id_2, section_id, boundary_position, boundary_type, boundary_status, paleo_lat, paleo_lng, ref_id) VALUES - (%(id)s, %(t1)s, %(t1_prop)s, %(t1_age)s, %(unit_id)s, %(unit_id_2)s, %(section_id)s, %(boundary_position)s, %(boundary_type)s, %(boundary_status)s, %(paleo_lat)s, %(paleo_lng)s, %(ref_id)s); diff --git a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_boundaries/3-index.sql deleted file mode 100644 index ddc31173..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_boundaries/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE INDEX on macrostrat.unit_boundaries (t1); -CREATE INDEX on macrostrat.unit_boundaries (unit_id); -CREATE INDEX on macrostrat.unit_boundaries (unit_id_2); -CREATE INDEX on macrostrat.unit_boundaries (section_id); \ No newline at end of file diff --git a/cli/macrostrat/cli/commands/table_meta/unit_econs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_econs/0-dump.sql deleted file mode 100644 index 7664ffb7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_econs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, unit_id, econ_id, ref_id, date_mod -FROM unit_econs - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_econs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_econs/1-create.sql deleted file mode 100644 index 8c3b4bd7..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_econs/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.unit_econs_new ( - id integer NOT NULL PRIMARY KEY, - unit_id integer, - econ_id integer, - ref_id integer, - date_mod text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_econs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_econs/2-insert.sql deleted file mode 100644 index 36e2d364..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_econs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.unit_econs_new (id, unit_id, econ_id, ref_id, date_mod) VALUES (%(id)s, %(unit_id)s, %(econ_id)s, %(ref_id)s, %(date_mod)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_econs/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_econs/3-index.sql deleted file mode 100644 index 1f4c2a2b..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_econs/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.unit_econs_new (econ_id); -CREATE INDEX ON macrostrat.unit_econs_new (unit_id); -CREATE INDEX ON macrostrat.unit_econs_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_environs/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_environs/0-dump.sql deleted file mode 100644 index b6149ce5..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_environs/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, unit_id, environ_id, ref_id, date_mod -FROM unit_environs - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_environs/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_environs/1-create.sql deleted file mode 100644 index e76de328..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_environs/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.unit_environs_new ( - id integer NOT NULL PRIMARY KEY, - unit_id integer, - environ_id integer, - ref_id integer, - date_mod text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_environs/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_environs/2-insert.sql deleted file mode 100644 index 99546f7e..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_environs/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.unit_environs_new (id, unit_id, environ_id, ref_id, date_mod) VALUES (%(id)s, %(unit_id)s, %(environ_id)s, %(ref_id)s, %(date_mod)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_environs/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_environs/3-index.sql deleted file mode 100644 index 490afbff..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_environs/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.unit_environs_new (environ_id); -CREATE INDEX ON macrostrat.unit_environs_new (unit_id); -CREATE INDEX ON macrostrat.unit_environs_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/0-dump.sql deleted file mode 100644 index 28d94034..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, unit_lith_id, lith_att_id, ref_id, date_mod -FROM unit_liths_atts - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/1-create.sql deleted file mode 100644 index 498a22dd..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/1-create.sql +++ /dev/null @@ -1,9 +0,0 @@ - -CREATE TABLE macrostrat.unit_lith_atts_new ( - id integer NOT NULL PRIMARY KEY, - unit_lith_id integer, - lith_att_id integer, - ref_id integer, - date_mod text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/2-insert.sql deleted file mode 100644 index c4b8da45..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.unit_lith_atts_new (id, unit_lith_id, lith_att_id, ref_id, date_mod) VALUES (%(id)s, %(unit_lith_id)s, %(lith_att_id)s, %(ref_id)s, %(date_mod)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/3-index.sql deleted file mode 100644 index cab49ff8..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_lith_atts/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.unit_lith_atts_new (unit_lith_id); -CREATE INDEX ON macrostrat.unit_lith_atts_new (lith_att_id); -CREATE INDEX ON macrostrat.unit_lith_atts_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_liths/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_liths/0-dump.sql deleted file mode 100644 index f5e01f1c..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_liths/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, lith_id, unit_id, prop, dom, comp_prop, mod_prop, toc, ref_id, date_mod -FROM unit_liths - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_liths/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_liths/1-create.sql deleted file mode 100644 index f09ba911..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_liths/1-create.sql +++ /dev/null @@ -1,14 +0,0 @@ - -CREATE TABLE macrostrat.unit_liths_new ( - id integer NOT NULL PRIMARY KEY, - lith_id integer, - unit_id integer, - prop text, - dom text, - comp_prop numeric, - mod_prop numeric, - toc numeric, - ref_id integer, - date_mod text -); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_liths/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_liths/2-insert.sql deleted file mode 100644 index b03c7509..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_liths/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.unit_liths_new (id, lith_id, unit_id, prop, dom, comp_prop, mod_prop, toc, ref_id, date_mod) VALUES (%(id)s, %(lith_id)s, %(unit_id)s, %(prop)s, %(dom)s, %(comp_prop)s, %(mod_prop)s, %(toc)s, %(ref_id)s, %(date_mod)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_liths/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_liths/3-index.sql deleted file mode 100644 index 77016fe3..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_liths/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.unit_liths_new (lith_id); -CREATE INDEX ON macrostrat.unit_liths_new (unit_id); -CREATE INDEX ON macrostrat.unit_liths_new (ref_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/unit_strat_names/0-dump.sql deleted file mode 100644 index f466074d..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, unit_id, strat_name_id -FROM unit_strat_names - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/1-create.sql b/cli/macrostrat/cli/commands/table_meta/unit_strat_names/1-create.sql deleted file mode 100644 index 3cf2ee52..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/1-create.sql +++ /dev/null @@ -1,7 +0,0 @@ - -CREATE TABLE macrostrat.unit_strat_names_new ( - id serial PRIMARY KEY NOT NULL, - unit_id integer NOT NULL, - strat_name_id integer NOT NULL -); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/unit_strat_names/2-insert.sql deleted file mode 100644 index 0e936bc2..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.unit_strat_names_new (id, unit_id, strat_name_id) VALUES (%(id)s, %(unit_id)s, %(strat_name_id)s); - diff --git a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/3-index.sql b/cli/macrostrat/cli/commands/table_meta/unit_strat_names/3-index.sql deleted file mode 100644 index bb35d720..00000000 --- a/cli/macrostrat/cli/commands/table_meta/unit_strat_names/3-index.sql +++ /dev/null @@ -1,4 +0,0 @@ - -CREATE INDEX ON macrostrat.unit_strat_names_new (unit_id); -CREATE INDEX ON macrostrat.unit_strat_names_new (strat_name_id); - diff --git a/cli/macrostrat/cli/commands/table_meta/units_sections/0-dump.sql b/cli/macrostrat/cli/commands/table_meta/units_sections/0-dump.sql deleted file mode 100644 index be95112e..00000000 --- a/cli/macrostrat/cli/commands/table_meta/units_sections/0-dump.sql +++ /dev/null @@ -1,4 +0,0 @@ - -SELECT id, unit_id, section_id, col_id -FROM units_sections - diff --git a/cli/macrostrat/cli/commands/table_meta/units_sections/1-create.sql b/cli/macrostrat/cli/commands/table_meta/units_sections/1-create.sql deleted file mode 100644 index 734d2baa..00000000 --- a/cli/macrostrat/cli/commands/table_meta/units_sections/1-create.sql +++ /dev/null @@ -1,8 +0,0 @@ - -CREATE TABLE macrostrat.units_sections_new ( - id serial PRIMARY KEY NOT NULL, - unit_id integer NOT NULL, - section_id integer NOT NULL, - col_id integer NOT NULL -); - diff --git a/cli/macrostrat/cli/commands/table_meta/units_sections/2-insert.sql b/cli/macrostrat/cli/commands/table_meta/units_sections/2-insert.sql deleted file mode 100644 index 3efe3866..00000000 --- a/cli/macrostrat/cli/commands/table_meta/units_sections/2-insert.sql +++ /dev/null @@ -1,3 +0,0 @@ - -INSERT INTO macrostrat.units_sections_new (id, unit_id, section_id, col_id) VALUES (%(id)s, %(unit_id)s, %(section_id)s, %(col_id)s) - diff --git a/cli/macrostrat/cli/commands/table_meta/units_sections/3-index.sql b/cli/macrostrat/cli/commands/table_meta/units_sections/3-index.sql deleted file mode 100644 index 9091bb6d..00000000 --- a/cli/macrostrat/cli/commands/table_meta/units_sections/3-index.sql +++ /dev/null @@ -1,5 +0,0 @@ - -CREATE INDEX ON macrostrat.units_sections_new (unit_id); -CREATE INDEX ON macrostrat.units_sections_new (section_id); -CREATE INDEX ON macrostrat.units_sections_new (col_id); - diff --git a/cli/macrostrat/cli/database/__init__.py b/cli/macrostrat/cli/database/__init__.py index 85cd2f57..c34dfaec 100644 --- a/cli/macrostrat/cli/database/__init__.py +++ b/cli/macrostrat/cli/database/__init__.py @@ -2,27 +2,25 @@ from pathlib import Path from sys import exit, stderr, stdin from typing import Any, Callable -from urllib.parse import quote import typer from macrostrat.database import Database from macrostrat.utils.shell import run from pydantic import BaseModel from rich import print -from sqlalchemy import create_engine, text -from sqlalchemy_utils import create_database +from sqlalchemy import text from typer import Argument, Option from .migrations import run_migrations +from .utils import engine_for_db_name + from macrostrat.core import MacrostratSubsystem, app from macrostrat.core.utils import is_pg_url from .._dev.utils import ( - _create_database_if_not_exists, - _docker_local_run_args, raw_database_url, ) -from ._legacy import * +from ._legacy import get_db __here__ = Path(__file__).parent fixtures_dir = __here__.parent / "fixtures" @@ -213,7 +211,7 @@ def dump( db_container = app.settings.get("pg_database_container", "postgres:15") - engine = _engine_for_db_name(database) + engine = engine_for_db_name(database) args = ctx.args custom_format = True @@ -243,7 +241,7 @@ def restore( db_container = app.settings.get("pg_database_container", "postgres:15") - engine = _engine_for_db_name(database) + engine = engine_for_db_name(database) args = [] if jobs is not None: @@ -258,14 +256,6 @@ def restore( ) -def _engine_for_db_name(name: str | None): - engine = get_db().engine - if name is None: - return engine - url = engine.url.set(database=name) - return create_engine(url) - - @db_app.command(name="tables") def list_tables(ctx: typer.Context, database: str = Argument(None), schema: str = None): """List tables in the database""" @@ -281,7 +271,7 @@ def list_tables(ctx: typer.Context, database: str = Argument(None), schema: str sql += "\nORDER BY table_schema, table_name;" - engine = _engine_for_db_name(database) + engine = engine_for_db_name(database) print( f"[dim]Tables in database: [bold cyan]{engine.url.database}[/]\n", file=stderr @@ -409,42 +399,3 @@ def field_title(name): db_app.command(name="migrations", rich_help_panel="Schema management")(run_migrations) - - -@db_app.command( - name="import-mariadb", rich_help_panel="Schema management", deprecated=True -) -def import_legacy(): - """Import legacy MariaDB database to PostgreSQL using pgloader""" - # Run pgloader in docker - - cfg = app.settings - - args = _docker_local_run_args(postgres_container="dimitri/pgloader") - - # Get the database URL - db = get_db() - url = db.engine.url - url = url.set(database="macrostrat_v1") - - _create_database_if_not_exists(url, create=True) - - pg_url = str(url) - # Repl - - pg_url = cfg.get("pg_database", None) - - url = pg_url + "_v1" - - dburl = cfg.get("mysql_database", None) - if dburl is None: - raise Exception("No MariaDB database URL available in configuration") - - run( - *args, - "pgloader", - "--with", - "prefetch rows = 1000", - str(dburl), - str(url), - ) diff --git a/cli/macrostrat/cli/database/mariadb/__init__.py b/cli/macrostrat/cli/database/mariadb/__init__.py new file mode 100644 index 00000000..bc26e499 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/__init__.py @@ -0,0 +1,89 @@ +from pathlib import Path +from sys import stdin + +from macrostrat.utils.shell import run +from sqlalchemy.engine.url import URL +from typer import Typer, Context, Argument + +from .restore import restore_mariadb, dump_mariadb +from .utils import build_connection_args, mariadb_engine +from ..utils import docker_internal_url +from .postgresql_migration import migrate_mariadb_to_postgresql + + +app = Typer(no_args_is_help=True) + +mariadb_container = "mariadb:10.10" + +# TODO: Adjust Typer context to ignore unconsumed arguments or arguments after "--" + + +@app.command( + name="cli", + add_help_option=False, + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def cli_command(ctx: Context): + """Run the MariaDB CLI against the Macrostrat database.""" + from macrostrat.core.config import mysql_database + + _database: URL = docker_internal_url(mysql_database) + + flags = [ + "-i", + "--rm", + ] + + if len(ctx.args) == 0 and stdin.isatty(): + flags.append("-t") + + run( + "docker", + "run", + *flags, + mariadb_container, + "mariadb", + *build_connection_args(_database), + *ctx.args, + ) + + +@app.command( + "dump", + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def dump_command( + ctx: Context, + output: Path = Argument(None, help="Path to the dump file"), + database: str = Argument(None, help="Database to dump"), +): + """Dump a MariaDB database to a file.""" + engine = mariadb_engine(database) + + if output is None: + output = Path("/dev/stdout") + + dump_mariadb(engine, output, *ctx.args, container=mariadb_container) + + +@app.command("restore") +def restore_command( + input: str = Argument(None, help="Path to the dump file or stream"), + database: str = Argument(None, help="Database to restore to"), + *, + create: bool = False, + overwrite: bool = False, +): + """Restore a MariaDB database from a dump file or stream.""" + engine = mariadb_engine(database) + + restore_mariadb( + input, + engine, + create=create, + overwrite=overwrite, + container=mariadb_container, + ) + + +app.command("migrate-to-postgres")(migrate_mariadb_to_postgresql) diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/__init__.py b/cli/macrostrat/cli/database/mariadb/postgresql_migration/__init__.py new file mode 100644 index 00000000..2450e481 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/__init__.py @@ -0,0 +1,313 @@ +from pathlib import Path +from textwrap import dedent + +from macrostrat.database import database_exists, create_database, drop_database +from macrostrat.database.utils import run_sql, run_query +from macrostrat.utils import get_logger +from macrostrat.utils.shell import run +from sqlalchemy import text, create_engine, inspect +from sqlalchemy.engine import Engine, make_url +from macrostrat.core.config import settings +from macrostrat.core import app +from .db_changes import ( + get_data_counts_maria, + get_data_counts_pg, + compare_data_counts, + find_row_variances, + find_col_variances, +) +from psycopg2.sql import Identifier +from ..restore import copy_mariadb_database +from ..utils import mariadb_engine +from ..._legacy import get_db +from ...utils import docker_internal_url, pg_temp_user +from ...._dev.utils import raw_database_url + +__here__ = Path(__file__).parent + +log = get_logger(__name__) + +from enum import Enum + + +class MariaDBMigrationStep(Enum): + COPY_MARIADB = "copy-mariadb" + PGLOADER = "pgloader" + CHECK_DATA = "check-data" + FINALIZE = "finalize" + + +_all_steps = { + MariaDBMigrationStep.COPY_MARIADB, + MariaDBMigrationStep.PGLOADER, + MariaDBMigrationStep.CHECK_DATA, + MariaDBMigrationStep.FINALIZE, +} + + +def migrate_mariadb_to_postgresql( + overwrite: bool = False, step: list[MariaDBMigrationStep] = None +): + """Migrate the legacy Macrostrat database from MariaDB to PostgreSQL.""" + + # Get the default MariaDB and PostgreSQL engines from the Macrostrat app's + # configuration (macrostrat.toml). + maria_engine = mariadb_engine() + pg_engine = get_db().engine + temp_db_name = "macrostrat_temp" + maria_temp_engine = create_engine(maria_engine.url.set(database=temp_db_name)) + #pg_temp_engine = create_engine(make_url(settings.pgloader_target_database)) + + # Destination schemas in the PostgreSQL database + temp_schema = temp_db_name + final_schema = "macrostrat" + + steps: set[MariaDBMigrationStep] = _all_steps + if step is not None and len(step) > 0: + steps = set(step) + + if MariaDBMigrationStep.COPY_MARIADB in steps: + copy_mariadb_database(maria_engine, maria_temp_engine, overwrite=overwrite) + + if MariaDBMigrationStep.PGLOADER in steps: + pgloader(maria_temp_engine, pg_engine, temp_schema, overwrite=overwrite) + + if MariaDBMigrationStep.CHECK_DATA in steps: + # NOTE: the temp schema and the final schema must be provided + should_proceed = compare_row_counts(maria_temp_engine, pg_engine, temp_schema) + if should_proceed: + raise ValueError("Data comparison failed. Aborting migration.") + else: + print("\ncheck-data completed!") + + + if MariaDBMigrationStep.FINALIZE in steps: + should_proceed = preserve_macrostrat_data(pg_engine) + if should_proceed: + raise NotImplementedError("Copy to macrostrat schema not yet implemented") + else: + print("finalize completed!") + +def pgloader(source: Engine, dest: Engine, target_schema: str, overwrite: bool = False): + _build_pgloader() + + if target_schema != source.url.database: + raise ValueError( + "The target schema must be the same as the source database name" + ) + + pgloader_pre_script(source) + _schema = Identifier(target_schema) + + if overwrite: + run_sql( + dest, + """ + DROP SCHEMA IF EXISTS {schema} CASCADE; + CREATE SCHEMA {schema}; + """, + dict(schema=_schema), + ) + + username = "maria_migrate" + with pg_temp_user(dest, username, overwrite=overwrite) as pg_temp: + # Create a temporary user that PGLoader can use to connect to the PostgreSQL database + # and create the temporary schema. + run_sql( + dest, + "GRANT ALL PRIVILEGES ON SCHEMA {schema} TO {user}", + dict( + schema=_schema, + user=Identifier(username), + ), + ) + _run_pgloader(source, pg_temp) + pgloader_post_script(pg_temp) + + +def schema_exists(engine: Engine, schema: str): + return run_query( + engine, + "SELECT 1 FROM information_schema.schemata WHERE schema_name = :schema_name", + dict(schema=schema), + ).scalar() + + +def pgloader_pre_script(engine: Engine): + assert engine.url.drivername.startswith("mysql") + pre_script = __here__ / "pgloader-pre-script.sql" + run_sql(engine, pre_script) + + +def pgloader_post_script(engine: Engine): + app.console.print("\n[bold]Running post-migration script[/]") + assert engine.url.drivername.startswith("postgres") + post_script = __here__ / "pgloader-post-script.sql" + run_sql(engine, post_script) + + +def _run_pgloader(source: Engine, dest: Engine): + """ + Command terminal to run pgloader. Ensure Docker app is running. + """ + db_exists = database_exists(dest.url) + if not db_exists: + header("Creating PostgreSQL database") + create_database(dest.url) + + header("Running pgloader") + + # PyMySQL is not installed in the pgloader image, so we need to use the mysql client + # to connect to the MariaDB database. + source_url = source.url.set(drivername="mysql") + + run( + "docker", + "run", + "-i", + "--rm", + "pgloader-runner", + "--with", + "prefetch rows = 1000", + "--verbose", + raw_database_url(docker_internal_url(source_url)), + raw_database_url(docker_internal_url(dest.url)) + "?sslmode=prefer", + ) + + +def _build_pgloader(): + header("Building pgloader-runner Docker image") + + dockerfile = dedent( + """FROM dimitri/pgloader:latest + RUN apt-get update && apt-get install -y postgresql-client ca-certificates && rm -rf /var/lib/apt/lists/* + ENTRYPOINT ["pgloader"] + """ + ) + + run( + "docker", + "build", + "-t", + "pgloader-runner:latest", + "-", + input=dockerfile.encode("utf-8"), + ) + + +def compare_row_counts(maria: Engine, pg_engine: Engine, schema): + + console = app.console + + maria_rows, maria_columns = get_data_counts_maria(maria) + pg_macrostrat_temp_rows, pg_macrostrat_temp_columns = get_data_counts_pg( + pg_engine, schema + ) + + db1 = db_identifier(maria) + db2 = schema + db3 = db_identifier(pg_engine) + + header(f"\n\nComparing [cyan]{db1}[/] to [cyan]{db2}[/].") + + row_variance, column_variance = compare_data_counts( + maria_rows, + pg_macrostrat_temp_rows, + maria_columns, + pg_macrostrat_temp_columns, + db1, + db2, + ) + + pg_rows, pg_columns = get_data_counts_pg(pg_engine, "macrostrat") + + header(f"\n\nComparing [cyan]{db2}[/] to [cyan]{db3}[/].") + + row_variance_two, column_variance_two = compare_data_counts( + pg_macrostrat_temp_rows, + pg_rows, + pg_macrostrat_temp_columns, + pg_columns, + db2, + db3, + ) + # reset() + # df, df_two = find_row_variances(pg_db_name, pg_db_name, pg_db_name_two, maria_db_name_two, + # pg_user, pg_pass_new, 'cols') + + tables = [ + "col_refs", + "lookup_unit_attrs_api", + "lookup_unit_intervals", + "strat_names_meta", + "sections", + "unit_econs", + "lookup_strat_names", + "measures", + "projects", + "timescales", + "strat_tree", + "refs", + "unit_liths", + "lookup_units", + "measurements", + "units", + "autocomplete", + "col_areas", + "unit_strat_names", + "unit_environs", + "cols", + "intervals", + "lith_atts", + "timescales_intervals", + "unit_boundaries", + "econs", + "environs", + "units_sections", + "unit_measures", + "strat_names", + "lookup_unit_liths", + "liths", + "concepts_places", + "strat_names_places", + "col_groups", + "measuremeta", + "places", + ] + find_row_variances( + pg_engine.url.database, + pg_engine.url.database, + "macrostrat_temp", + pg_engine.url.username, + pg_engine.url.password, + tables, + pg_engine, + ) + find_col_variances( + pg_engine.url.database, + pg_engine.url.database, + "macrostrat_temp", + pg_engine.url.username, + pg_engine.url.password, + tables, + pg_engine, + ) + +def preserve_macrostrat_data(engine: Engine): + app.console.print("\n[bold]Running script[/]") + assert engine.url.drivername.startswith("postgres") + preserve_data = __here__ / "preserve-macrostrat-data.sql" + run_sql(engine, preserve_data) + +def db_identifier(engine: Engine): + driver = engine.url.drivername + if driver.startswith("postgresql"): + driver = "PostgreSQL" + elif driver.startswith("mysql"): + driver = "MariaDB" + + return f"{engine.url.database} ({driver})" + + +def header(text): + app.console.print(f"\n[bold]{text}[/]\n") diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/db_changes.py b/cli/macrostrat/cli/database/mariadb/postgresql_migration/db_changes.py new file mode 100644 index 00000000..1e5b3cf1 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/db_changes.py @@ -0,0 +1,327 @@ +""" +Script to output dataframes for comparing data between two databases and tables. +""" + +import pandas as pd +from macrostrat.database import run_query +from psycopg2.sql import Identifier +from sqlalchemy import create_engine, text, inspect +from sqlalchemy.engine import Engine +from macrostrat.core import app + +console = app.console + + +def get_data_counts_maria(engine: Engine): + db_name = engine.url.database + maria_rows = {} + maria_columns = {} + with engine.connect() as conn: + row_result = run_query( + conn, + "SELECT table_name FROM information_schema.tables WHERE table_schema = :table_schema AND table_type = 'BASE TABLE'", + {"table_schema": db_name}, + ) + maria_tables = [row[0] for row in row_result] + for table in maria_tables: + row_result = run_query(conn, f"SELECT COUNT(*) FROM {table}") + row_count = row_result.scalar() + maria_rows[table.lower()] = row_count + column_result = run_query( + conn, + "SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = :table_schema AND table_name = :table_name", + dict(table_schema=db_name, table_name=table), + ) + column_count = column_result.scalar() + maria_columns[table.lower()] = column_count + engine.dispose() + return maria_rows, maria_columns + + +def get_data_counts_pg(engine: Engine, schema): + database_name = engine.url.database + pg_rows = {} + pg_columns = {} + with engine.connect() as conn: + table_result = run_query( + conn, + """ + SELECT table_name FROM information_schema.tables + WHERE table_catalog = :table_catalog + AND table_type = 'BASE TABLE' AND table_schema = :table_schema + """, + dict(table_catalog=database_name, table_schema=schema), + ) + pg_tables = [row[0] for row in table_result] + + for table in pg_tables: + row_result = run_query( + conn, + f"SELECT COUNT(*) FROM {schema}.{table}", + ) + row_count = row_result.scalar() + pg_rows[table.lower()] = row_count + + column_result = run_query( + conn, + """ + SELECT COUNT(*) FROM information_schema.columns + WHERE table_catalog = :table_catalog + AND table_schema = :schema + AND table_name = :table + """, + dict(table_catalog=database_name, schema=schema, table=table), + ) + column_count = column_result.scalar() + pg_columns[table.lower()] = column_count + engine.dispose() + return pg_rows, pg_columns + + +def compare_data_counts(db1_rows, db2_rows, db1_columns, db2_columns, db1, db2): + """ + Compares the data counts between tables, rows, and columns that vary between any two db's + """ + + db1_rows_not_in_db2 = { + table_name: (db1_rows[table_name], 0) + for table_name in db1_rows + if table_name not in db2_rows + } + db2_rows_not_in_db1 = { + table_name: (0, db2_rows[table_name]) + for table_name in db2_rows + if table_name not in db1_rows + } + db1_cols_not_in_db2 = { + table_name: (db1_columns[table_name], 0) + for table_name in db1_columns + if table_name not in db2_columns + } + db2_cols_not_in_db1 = { + table_name: (0, db2_columns[table_name]) + for table_name in db2_columns + if table_name not in db1_columns + } + + console.print("\n[bold]Checking table counts...") + + if len(db1_rows_not_in_db2) == 0 and len(db2_rows_not_in_db1) == 0: + success(f"All tables exist in both {db1} and {db2}.") + else: + count = 0 + maria_tables = [ + 'unit_notes', + 'col_areas_6april2016', + 'col_equiv', + 'col_notes', + 'interval_boundaries', + 'interval_boundaries_scratch', + 'measuremeta_cols', + 'minerals', + 'offshore_baggage', + 'offshore_baggage_units', + 'offshore_fossils', + 'pbdb_matches', + 'rockd_features', + 'ronov_sediment', + 'stats', + 'strat_names_lookup', + 'structures', + 'structure_atts', + 'tectonics', + 'temp_areas', + 'uniquedatafiles2', + 'units_datafiles', + 'unit_boundaries_backup', + 'unit_boundaries_scratch', + 'unit_boundaries_scratch_old', + 'unit_contacts', + 'unit_dates', + 'unit_measures_pbdb', + 'unit_seq_strat', + 'unit_tectonics', + 'canada_lexicon_dump', + 'colors', + 'lookup_measurements', + 'offshore_sections', + 'offshore_hole_ages', + 'offshore_sites', + 'pbdb_intervals', + 'pbdb_liths', + 'unit_equiv', + 'unit_liths_atts' + ] + + if list(db1_rows_not_in_db2.keys()) == maria_tables: + success(f"{len(db1_rows_not_in_db2)} {db1} tables copied over from MariaDB that do not exist in {db2}. This confirms data retention!") + console.print( + [key for key in db1_rows_not_in_db2], + ) + elif len(db1_rows_not_in_db2) > 0 and list(db1_rows_not_in_db2.keys()) != maria_tables: + error(f"{len(db1_rows_not_in_db2)} {db1} tables not found in {db2}:") + console.print( + [key for key in db1_rows_not_in_db2], + ) + if list(db2_rows_not_in_db1.keys()) == ['strat_name_footprints', 'grainsize', 'pbdb_collections', 'pbdb_collections_strat_names']: + success(f"{len(db2_rows_not_in_db1)} macrostrat (PostgreSQL) tables succesfully copied into {db2} to retain data!") + console.print( + [key for key in db2_rows_not_in_db1], + ) + if list(db2_rows_not_in_db1.keys()) == ['temp_rocks', 'temp_names', 'unit_lith_atts']: + success( + f"{len(db2_rows_not_in_db1)} {db2} tables did not copy into {db1}. These tables are irrelevant and do not need to be retained.") + console.print( + [key for key in db2_rows_not_in_db1], + ) + elif len(db2_rows_not_in_db1) > 0 and list(db2_rows_not_in_db1.keys()) != ['strat_name_footprints', 'grainsize', 'pbdb_collections', 'pbdb_collections_strat_names']: + error(f"{len(db2_rows_not_in_db1)} {db2} tables not found in {db1}:") + console.print( + [key for key in db2_rows_not_in_db1], + ) + + console.print("\n[bold]Checking row counts...") + + row_count_difference = { + key: (db1_rows[key], db2_rows[key]) + for key in db1_rows + if key in db2_rows and db1_rows[key] != db2_rows[key] + } + # row_count_difference.update(db1_rows_not_in_db2) + # row_count_difference.update(db2_rows_not_in_db1) + + col_count_difference = { + key: (db1_columns[key], db2_columns[key]) + for key in db1_columns + if key in db2_columns and db1_columns[key] != db2_columns[key] + } + # col_count_difference.update(db1_cols_not_in_db2) + # col_count_difference.update(db2_cols_not_in_db1) + + if len(row_count_difference) == 0: + success(f"All row counts in all tables are the same in {db1} and {db2}!") + elif db1 == 'macrostrat_temp' and db2 == 'macrostrat (PostgreSQL)' and len(row_count_difference) == 26: + success( + f"Row counts are greater in {db1} rather than {db2} for {len(row_count_difference)} tables, indicating data retention from Mariadb!" + ) + print_counts(row_count_difference) + + else: + error( + f"Row count differences for {len(row_count_difference)} tables in {db1} and {db2} databases" + ) + print_counts(row_count_difference) + + + + console.print("\n[bold]Checking column counts...") + + if len(col_count_difference) == 0: + success(f"All column counts in all tables are the same in {db1} and {db2}!\n") + + if db1 == "macrostrat_temp (MariaDB)" and db2 == "macrostrat_temp" and list(col_count_difference.keys()) == ['lookup_unit_intervals', 'col_areas', 'cols', 'intervals', 'measuremeta']: + success( + f"Columns for {len(col_count_difference)} tables successfully copied over from macrostrat (PostgreSQL) into {db2}, to retain data!" + ) + print_col_counts(col_count_difference) + elif list(col_count_difference.keys()) == ['cols', 'col_areas', 'environs','intervals' ,'lith_atts' ,'measures', 'sections', 'strat_names', 'units', 'unit_environs', 'unit_strat_names', 'lookup_strat_names', 'strat_tree']: + success( + f"Columns for {len(col_count_difference)-1} are greater in {db1} rather than {db2}. This indicates data retention!" + ) + print_counts(col_count_difference) + + else: + error( + f"Column count differences for {len(col_count_difference)} tables in {db1} and {db2} databases" + ) + print_counts(col_count_difference) + + return row_count_difference, col_count_difference + + +def print_counts(counts): + for key, (v1, v2) in counts.items(): + diff = v1 - v2 + col = "red" if diff < 0 else "green" + diff = f"[{col}]{diff:+8d}[/]" + + console.print(f"{key:30s} {v1:9d} {v2:9d} [dim]{diff}[/dim]") + +def print_col_counts(counts): + for key, (v1, v2) in counts.items(): + diff = v2 - v1 + col = "red" if diff < 0 else "green" + diff = f"[{col}]{diff:+8d}[/]" + + console.print(f"{key:30s} {v1:9d} {v2:9d} [dim]{diff}[/dim]") + +def error(message): + console.print(f"\n[red bold]ERROR:[red] {message}") + + +def success(message): + console.print(f"\n[green bold]SUCCESS:[green] {message}") + + +def find_row_variances( + database_name_one, + schema_one, + schema_two, + username, + password, + tables, + pg_engine +): + insp = inspect(pg_engine) + count = 0 + with pg_engine.connect() as conn: + for table in tables: + # Get the actual first column name for each table + columns = insp.get_columns(table, schema=schema_one) + first_column_name = columns[0]['name'] + query = f""" + SELECT COUNT(m.{first_column_name}) + FROM macrostrat.macrostrat.{table} m + RIGHT JOIN macrostrat.macrostrat_temp.{table} t ON m.{first_column_name} = t.{first_column_name} + WHERE t.{first_column_name} IS NULL; + """ + result = conn.execute(text(query)) + dict = {} + for row in result: + dict[table] = (row[0], 0) + + pg_engine.dispose() + print(dict) + print_counts(dict) + return + +def find_col_variances( + database_name_one, + schema_one, + schema_two, + username, + password, + tables, + pg_engine +): + + insp = inspect(pg_engine) + results = [] + for table in tables: + columns_one = insp.get_columns(table, schema=schema_one) + columns_two = insp.get_columns(table, schema=schema_two) + col_names_one = {col['name'] for col in columns_one} + col_names_two = {col['name'] for col in columns_two} + col_not_in_schema_two = col_names_one - col_names_two + ['units', 'cols' ] + if col_not_in_schema_two == {'notes'}: + success(f"Notes column exists {schema_one} but NOT in {schema_two} for {table}. {schema_two}.{table}.notes is its own table from Mariadb.") + if col_not_in_schema_two and col_not_in_schema_two != {'notes'}: + error(f"Columns that exist in {schema_one} but NOT in {schema_two} for {table}: {col_not_in_schema_two}") + else: + results.append(table) + + success(f"All columns in {schema_one} exist in {schema_two} for these tables: ") + print(results) + pg_engine.dispose() + return diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-post-script.sql b/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-post-script.sql new file mode 100644 index 00000000..4c89df18 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-post-script.sql @@ -0,0 +1,69 @@ +/* + The query + - Alters the MariaDB pbdb_matches table by adding a new column for the text data, + - sets the datatype of the new column data to WKT format, + - drops old geometry columns + - refreshes the database after pgloader + + */ + +CREATE EXTENSION IF NOT EXISTS postgis; + +SET search_path TO macrostrat_temp, public; + +ALTER TABLE macrostrat_temp.pbdb_matches ADD COLUMN coordinate geometry(Point, 4326); +UPDATE macrostrat_temp.pbdb_matches SET coordinate = ST_GeomFromText(coordinate_point_text, 4326); +ALTER TABLE macrostrat_temp.pbdb_matches DROP COLUMN coordinate_point_text; +SELECT *FROM macrostrat_temp.pbdb_matches LIMIT 5; + +ALTER TABLE macrostrat_temp.places ADD COLUMN geom geometry; +UPDATE macrostrat_temp.places SET geom = ST_GeomFromText(geom_text, 4326); + +ALTER TABLE macrostrat_temp.places DROP COLUMN geom_text; +SELECT *FROM macrostrat_temp.places LIMIT 5; + +ALTER TABLE macrostrat_temp.refs ADD COLUMN rgeom geometry; +UPDATE macrostrat_temp.refs SET rgeom = ST_GeomFromText(rgeom_text, 4326); +ALTER TABLE macrostrat_temp.refs DROP COLUMN rgeom_text; +SELECT *FROM macrostrat_temp.refs LIMIT 5; + +ALTER TABLE macrostrat_temp.cols ADD COLUMN coordinate geometry; +UPDATE macrostrat_temp.cols SET coordinate = ST_GeomFromText(coordinate_text, 4326); +ALTER TABLE macrostrat_temp.cols DROP COLUMN coordinate_text; +SELECT *FROM macrostrat_temp.cols LIMIT 5; + +ALTER TABLE macrostrat_temp.col_areas ADD COLUMN col_area geometry; +UPDATE macrostrat_temp.col_areas SET col_area = ST_GeomFromText(col_area_text, 4326); +ALTER TABLE macrostrat_temp.col_areas DROP COLUMN col_area_text; +SELECT *FROM macrostrat_temp.col_areas LIMIT 5; + +ALTER TABLE macrostrat_temp.col_areas_6April2016 ADD COLUMN col_area geometry; +UPDATE macrostrat_temp.col_areas_6April2016 SET col_area = ST_GeomFromText(col_area_text, 4326); +ALTER TABLE macrostrat_temp.col_areas_6April2016 DROP COLUMN col_area_text; +SELECT *FROM macrostrat_temp.col_areas_6April2016 LIMIT 5; + +--added query below since column exists in macrostrat and not in macrostrat_temp. +ALTER TABLE macrostrat_temp.measuremeta ADD COLUMN geometry geometry(Point, 4326); +UPDATE macrostrat_temp.measuremeta SET geometry = ST_SetSRID(ST_makepoint(lng, lat), 4326); + +ALTER TABLE macrostrat.macrostrat_temp.lookup_unit_intervals ADD COLUMN best_interval_id INTEGER; + +--added query below since column exists in macrostrat and not in macrostrat_temp. + +ALTER TABLE macrostrat_temp.col_areas ADD COLUMN wkt text; +UPDATE macrostrat_temp.col_areas SET wkt = ST_AsText(col_area); + +--added query below since column exists in macrostrat and not in macrostrat_temp. + +ALTER TABLE macrostrat_temp.cols ADD COLUMN wkt text; +ALTER TABLE macrostrat_temp.cols ADD COLUMN poly_geom geometry; +UPDATE macrostrat_temp.cols SET wkt = ST_AsText(coordinate); +UPDATE macrostrat_temp.cols c SET poly_geom = a.col_area +FROM macrostrat_temp.col_areas a WHERE c.id = a.col_id; +UPDATE macrostrat_temp.cols SET poly_geom = ST_SetSRID(poly_geom, 4326); + + +--add rank column since this column only exists in macrostrat and not macrostrat_temp + +ALTER TABLE macrostrat_temp.intervals ADD COLUMN rank integer; + diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-pre-script.sql b/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-pre-script.sql new file mode 100644 index 00000000..52382b1e --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/pgloader-pre-script.sql @@ -0,0 +1,72 @@ +/* SQL script that + - alters the MariaDB tables by adding a new column for geom -> text data, + - sets the datatype of the new column data to WKT format, + - drops the old geometry column, + - adds default values for data formats that pgloader accepts + + NOTE: this runs in MariaDB, not PostgreSQL + */ + +ALTER TABLE macrostrat_temp.pbdb_matches ADD COLUMN coordinate_point_text TEXT; + +UPDATE macrostrat_temp.pbdb_matches SET coordinate_point_text = ST_AsText(coordinate); + +ALTER TABLE macrostrat_temp.pbdb_matches DROP COLUMN coordinate; + +UPDATE macrostrat_temp.pbdb_matches SET release_date = '2000-01-01' WHERE release_date = '0000-00-00 00:00:00'; + +ALTER TABLE macrostrat_temp.places ADD COLUMN geom_text LONGTEXT; + +UPDATE macrostrat_temp.places +SET geom_text = ST_AsText(geom); +ALTER TABLE macrostrat_temp.places DROP COLUMN geom; + + +--Added query below to match the PG macrostrat database. Will need to review the API to ensure it doesn't break. +--https://github.com/UW-Macrostrat/macrostrat/blob/7aefe2d0cc89a738b356ff444b7b3dd0fd85e607/cli/macrostrat/cli/commands/table_meta/strat_tree/0-dump.sql +ALTER TABLE macrostrat_temp.strat_tree RENAME COLUMN this_name TO parent; +ALTER TABLE macrostrat_temp.strat_tree RENAME COLUMN that_name TO child; + + + +ALTER TABLE macrostrat_temp.refs ADD COLUMN rgeom_text LONGTEXT; +UPDATE macrostrat_temp.refs +SET rgeom_text = ST_AsText(rgeom); +ALTER TABLE macrostrat_temp.refs DROP COLUMN rgeom; + +UPDATE unit_contacts +-- Enum data type can't be null so set to enum option 'below'. +SET contact = 'below' +WHERE contact = ''; + +UPDATE unit_contacts +-- enum data type can't be null so set to enum option 'above'. +SET old_contact = 'above' +WHERE old_contact = ''; + +ALTER TABLE macrostrat_temp.cols ADD COLUMN coordinate_text LONGTEXT; +UPDATE macrostrat_temp.cols +SET coordinate_text = ST_AsText(coordinate); + +ALTER TABLE macrostrat_temp.cols DROP COLUMN coordinate; + +UPDATE macrostrat_temp.cols +SET created = '2000-01-01' +WHERE created = '0000-00-00 00:00:00'; + +ALTER TABLE macrostrat_temp.col_areas ADD COLUMN col_area_text LONGTEXT; + +UPDATE macrostrat_temp.col_areas +SET col_areas.col_area_text = ST_AsText(col_area); + +ALTER TABLE macrostrat_temp.col_areas DROP COLUMN col_area; + +ALTER TABLE macrostrat_temp.col_areas_6April2016 ADD COLUMN col_area_text LONGTEXT; + +UPDATE macrostrat_temp.col_areas_6April2016 +SET col_areas_6April2016.col_area_text = ST_AsText(col_area); + +ALTER TABLE macrostrat_temp.col_areas_6April2016 DROP COLUMN col_area; + +UPDATE macrostrat_temp.liths +SET lith_group = NULL WHERE lith_group = ''; diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/preserve-macrostrat-data.sql b/cli/macrostrat/cli/database/mariadb/postgresql_migration/preserve-macrostrat-data.sql new file mode 100644 index 00000000..4b664354 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/preserve-macrostrat-data.sql @@ -0,0 +1,62 @@ +--This query adds columns and data that exist in macrostrat and not in macrostrat_temp + +--Used this script to add best_interval_id column into macrostrat_temp.lookup_unit_intervals +--https://github.com/UW-Macrostrat/macrostrat/blob/7aefe2d0cc89a738b356ff444b7b3dd0fd85e607/cli/macrostrat/cli/commands/table_meta/lookup_unit_intervals/4-process.sql#L21 + +WITH bests AS ( + select unit_id, + CASE + WHEN age_id > 0 THEN + age_id + WHEN epoch_id > 0 THEN + epoch_id + WHEN period_id > 0 THEN + period_id + WHEN era_id > 0 THEN + era_id + WHEN eon_id > 0 THEN + eon_id + ELSE + 0 + END + AS b_interval_id from macrostrat_temp.lookup_unit_intervals +) +UPDATE macrostrat_temp.lookup_unit_intervals lui +SET best_interval_id = b_interval_id +FROM bests +WHERE lui.unit_id = bests.unit_id; +/* +This query copies the table configuration and all data from macrostrat.macrostrat and inserts it +into the macrostrat.temp schema. This is to preserve the data that exists in macrostrat and NOT in +MariaDB before we run the migration. +*/ + +DO $$ +DECLARE + table_name text; + source_schema text := 'macrostrat'; + target_schema text := 'macrostrat_temp'; + tables text[] := ARRAY[ + 'strat_name_footprints', + 'grainsize', + 'pbdb_collections', + 'pbdb_collections_strat_names' + ]; +BEGIN + FOREACH table_name IN ARRAY tables + LOOP + EXECUTE format('CREATE TABLE %I.%I (LIKE %I.%I INCLUDING ALL)', target_schema, table_name, source_schema, table_name); + EXECUTE format('INSERT INTO %I.%I SELECT * FROM %I.%I', target_schema, table_name, source_schema, table_name); + END LOOP; +END $$; + + + +--from schlep scripts +UPDATE macrostrat_temp.intervals SET rank = 6 WHERE interval_type = 'age'; +UPDATE macrostrat_temp.intervals SET rank = 5 WHERE interval_type = 'epoch'; +UPDATE macrostrat_temp.intervals SET rank = 4 WHERE interval_type = 'period'; +UPDATE macrostrat_temp.intervals SET rank = 3 WHERE interval_type = 'era'; +UPDATE macrostrat_temp.intervals SET rank = 2 WHERE interval_type = 'eon'; +UPDATE macrostrat_temp.intervals SET rank = 1 WHERE interval_type = 'supereon'; +UPDATE macrostrat_temp.intervals SET rank = 0 WHERE rank IS NULL; \ No newline at end of file diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/requirements.txt b/cli/macrostrat/cli/database/mariadb/postgresql_migration/requirements.txt new file mode 100644 index 00000000..21dd2320 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/requirements.txt @@ -0,0 +1,4 @@ +SQLAlchemy==2.0.7 +PyMySQL==1.1.1 +psycopg2==2.9.9 +pandas diff --git a/cli/macrostrat/cli/database/mariadb/postgresql_migration/schlep-index.sql b/cli/macrostrat/cli/database/mariadb/postgresql_migration/schlep-index.sql new file mode 100644 index 00000000..065a1461 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/postgresql_migration/schlep-index.sql @@ -0,0 +1,179 @@ +--Consolidating all of the schelp index scripts here from https://github.com/UW-Macrostrat/macrostrat/blob/maria-migrate/cli/macrostrat/cli/commands/table_meta/ + +--autocomplete +CREATE INDEX ON macrostrat.autocomplete_new (id); +CREATE INDEX ON macrostrat.autocomplete_new (name); +CREATE INDEX ON macrostrat.autocomplete_new (type); +CREATE INDEX ON macrostrat.autocomplete_new (category); + +--col_areas +CREATE INDEX ON macrostrat.col_areas_new (col_id); +CREATE INDEX ON macrostrat.col_areas_new USING GIST (col_area); + +--col_groups +CREATE INDEX ON macrostrat.col_groups_new (id); + +--col_refs +CREATE INDEX ON macrostrat.col_refs_new (col_id); +CREATE INDEX ON macrostrat.col_refs_new (ref_id); + +--cols +CREATE INDEX ON macrostrat.cols_new (project_id); +CREATE INDEX ON macrostrat.cols_new USING GIST (coordinate); +CREATE INDEX ON macrostrat.cols_new USING GIST (poly_geom); +CREATE INDEX ON macrostrat.cols_new (col_group_id); +CREATE INDEX ON macrostrat.cols_new (status_code); + +--concepts_places +CREATE INDEX ON macrostrat.concepts_places_new (concept_id); +CREATE INDEX ON macrostrat.concepts_places_new (place_id); + +--econs + +--environs + +--intervals +CREATE INDEX ON macrostrat.intervals_new (id); +CREATE INDEX ON macrostrat.intervals_new (age_top); +CREATE INDEX ON macrostrat.intervals_new (age_bottom); +CREATE INDEX ON macrostrat.intervals_new (interval_type); +CREATE INDEX ON macrostrat.intervals_new (interval_name); + +--lith_atts +CREATE INDEX ON macrostrat.lith_atts_new (att_type); +CREATE INDEX ON macrostrat.lith_atts_new (lith_att); + +--liths +CREATE INDEX ON macrostrat.liths_new (lith); +CREATE INDEX ON macrostrat.liths_new (lith_class); +CREATE INDEX ON macrostrat.liths_new (lith_type); + +--lookup_strat_names +CREATE INDEX ON macrostrat.lookup_strat_names_new (strat_name_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (concept_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (bed_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (mbr_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (fm_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (gp_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (sgp_id); +CREATE INDEX ON macrostrat.lookup_strat_names_new (strat_name); + +--lookup_unit_attrs_api +CREATE INDEX ON macrostrat.lookup_unit_attrs_api_new (unit_id); + +--lookup_unit_intervals +CREATE INDEX ON macrostrat.lookup_unit_intervals_new (unit_id); +CREATE INDEX ON macrostrat.lookup_unit_intervals_new (best_interval_id); + +--lookup_unit_liths +CREATE INDEX ON macrostrat.lookup_unit_liths_new (unit_id); + +--lookup_units +CREATE INDEX ON macrostrat.lookup_units_new (project_id); +CREATE INDEX ON macrostrat.lookup_units_new (t_int); +CREATE INDEX ON macrostrat.lookup_units_new (b_int); + +--measurements +CREATE INDEX ON macrostrat.measurements_new (id); +CREATE INDEX ON macrostrat.measurements_new (measurement_class); +CREATE INDEX ON macrostrat.measurements_new (measurement_type); + +--measuremeta +CREATE INDEX ON macrostrat.measuremeta_new (lith_id); +CREATE INDEX ON macrostrat.measuremeta_new (ref_id); +CREATE INDEX ON macrostrat.measuremeta_new (lith_att_id); + +--measures +CREATE INDEX ON macrostrat.measures_new (measurement_id); +CREATE INDEX ON macrostrat.measures_new (measuremeta_id); + +--pbdb_collections +CREATE INDEX ON macrostrat.pbdb_collections_new (collection_no); +CREATE INDEX ON macrostrat.pbdb_collections_new (early_age); +CREATE INDEX ON macrostrat.pbdb_collections_new (late_age); +CREATE INDEX ON macrostrat.pbdb_collections_new USING GiST (geom); + +--places +CREATE INDEX ON macrostrat.places_new USING GiST (geom); + +--projects +CREATE INDEX ON macrostrat.projects_new (project); +CREATE INDEX ON macrostrat.projects_new (timescale_id); + +--refs +CREATE INDEX ON macrostrat.refs_new USING GiST (rgeom); + +--sections +CREATE INDEX ON macrostrat.sections_new(id); +CREATE INDEX ON macrostrat.sections_new(col_id); + +--strat_names +CREATE INDEX ON macrostrat.strat_names_new (strat_name); +CREATE INDEX ON macrostrat.strat_names_new (rank); +CREATE INDEX ON macrostrat.strat_names_new (ref_id); +CREATE INDEX ON macrostrat.strat_names_new (concept_id); + +--strat_names_meta +CREATE INDEX ON macrostrat.strat_names_meta_new (interval_id); +CREATE INDEX ON macrostrat.strat_names_meta_new (b_int); +CREATE INDEX ON macrostrat.strat_names_meta_new (t_int); +CREATE INDEX ON macrostrat.strat_names_meta_new (ref_id); + +--strat_names_places +CREATE INDEX ON macrostrat.strat_names_places_new (strat_name_id); +CREATE INDEX ON macrostrat.strat_names_places_new (place_id); + +--strat_tree +CREATE INDEX ON macrostrat.strat_tree_new (parent); +CREATE INDEX ON macrostrat.strat_tree_new (child); +CREATE INDEX ON macrostrat.strat_tree_new (ref_id); + +--timescales +CREATE INDEX ON macrostrat.timescales_new (timescale); +CREATE INDEX ON macrostrat.timescales_new (ref_id); + +--timescales_intervals +CREATE INDEX ON macrostrat.timescales_intervals_new (timescale_id); +CREATE INDEX ON macrostrat.timescales_intervals_new (interval_id); + +--unit_boundaries +CREATE INDEX on macrostrat.unit_boundaries (t1); +CREATE INDEX on macrostrat.unit_boundaries (unit_id); +CREATE INDEX on macrostrat.unit_boundaries (unit_id_2); +CREATE INDEX on macrostrat.unit_boundaries (section_id); + +--unit_econs +CREATE INDEX ON macrostrat.unit_econs_new (econ_id); +CREATE INDEX ON macrostrat.unit_econs_new (unit_id); +CREATE INDEX ON macrostrat.unit_econs_new (ref_id); + +--unit_environs +CREATE INDEX ON macrostrat.unit_environs_new (environ_id); +CREATE INDEX ON macrostrat.unit_environs_new (unit_id); +CREATE INDEX ON macrostrat.unit_environs_new (ref_id); + +--unit_lith_atts +CREATE INDEX ON macrostrat.unit_lith_atts_new (unit_lith_id); +CREATE INDEX ON macrostrat.unit_lith_atts_new (lith_att_id); +CREATE INDEX ON macrostrat.unit_lith_atts_new (ref_id); + +--unit_liths +CREATE INDEX ON macrostrat.unit_liths_new (lith_id); +CREATE INDEX ON macrostrat.unit_liths_new (unit_id); +CREATE INDEX ON macrostrat.unit_liths_new (ref_id); + +--unit_strat_names +CREATE INDEX ON macrostrat.unit_strat_names_new (unit_id); +CREATE INDEX ON macrostrat.unit_strat_names_new (strat_name_id); + +--units +CREATE INDEX ON macrostrat.units_new (section_id); +CREATE INDEX ON macrostrat.units_new (col_id); +CREATE INDEX ON macrostrat.units_new (strat_name); +CREATE INDEX ON macrostrat.units_new (color); + +--units_sections + +CREATE INDEX ON macrostrat.units_sections_new (unit_id); +CREATE INDEX ON macrostrat.units_sections_new (section_id); +CREATE INDEX ON macrostrat.units_sections_new (col_id); diff --git a/cli/macrostrat/cli/database/mariadb/restore.py b/cli/macrostrat/cli/database/mariadb/restore.py new file mode 100644 index 00000000..ee8441bf --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/restore.py @@ -0,0 +1,188 @@ +import asyncio +from pathlib import Path +from typing import Union +from sys import stdin + +from macrostrat.utils import get_logger +from macrostrat.database import database_exists +from rich.console import Console +from sqlalchemy.engine import Engine, URL +from macrostrat.core.exc import MacrostratError +import aiofiles +from tempfile import NamedTemporaryFile +from contextlib import contextmanager + +from .utils import build_connection_args, ParameterStyle +from ..utils import docker_internal_url + + +from ..._dev.utils import ( + _create_command, + _create_database_if_not_exists, +) +from ..._dev.stream_utils import ( + print_stream_progress, + print_stdout, + DecodingStreamReader, +) + +console = Console() + +log = get_logger(__name__) + + +def restore_mariadb(_input: Union[str, Path, None], engine: Engine, *args, **kwargs): + """Restore a MariaDB database from a dump file or stream""" + + if _input is not None: + if str(_input).startswith("http"): + raise NotImplementedError("http(s) restore not yet implemented") + + _input = Path(_input) + + if _input is None: + if stdin.isatty(): + raise MacrostratError("No input file specified") + + # Read from stdin + _input = Path("/dev/stdin") + + if not _input.is_file() and not _input.is_fifo(): + raise MacrostratError(f"{_input} is not a file") + + task = _restore_mariadb_from_file(_input, engine, *args, **kwargs) + asyncio.run(task) + + +def _log_command(url: URL, cmd: list[str]): + logged_cmd = " ".join(cmd) + if url.password: + logged_cmd = logged_cmd.replace(url.password, "***") + log.debug(logged_cmd) + return logged_cmd + + +async def _restore_mariadb(engine: Engine, *args, **kwargs): + """Load MariaDB dump (GZipped SQL file) into a database, using centrally managed credentials, + a Docker containerized `mariadb` client, and a streaming approach.""" + overwrite = kwargs.pop("overwrite", False) + create = kwargs.pop("create", overwrite) + container = kwargs.pop("container", "mariadb:10.10") + + _create_database_if_not_exists( + engine.url, create=create, allow_exists=False, overwrite=overwrite + ) + conn = build_connection_args(docker_internal_url(engine.url)) + + # Run mariadb in a local Docker container + # or another location, if more appropriate. Running on the remote + # host, if possible, is probably the fastest option. There should be + # multiple options ideally. + _cmd = _create_command( + "mariadb", + *conn, + *args, + container=container, + ) + + _log_command(engine.url, _cmd) + + return await asyncio.create_subprocess_exec( + *_cmd, + stdin=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + limit=1024 * 1024 * 1, # 1 MB windows + # Stdout to dev null + stdout=asyncio.subprocess.PIPE, + ) + + +async def _restore_mariadb_from_file(dumpfile: Path, engine: Engine, *args, **kwargs): + proc = await _restore_mariadb(engine, *args, **kwargs) + # Open dump file as an async stream + async with aiofiles.open(dumpfile, mode="rb") as source: + s1 = DecodingStreamReader(source) + await asyncio.gather( + print_stream_progress(s1, proc.stdin, prefix="Restored"), + print_stdout(proc.stderr), + ) + + +async def _dump_mariadb(engine: Engine, *args, **kwargs): + """Dump a MariaDB database to a stream""" + container = kwargs.pop("container", "mariadb:10.10") + stdout = kwargs.pop("stdout", asyncio.subprocess.PIPE) + + conn = build_connection_args( + docker_internal_url(engine.url), ParameterStyle.MySQLDump + ) + + _cmd = _create_command( + "mysqldump", + *conn, + *args, + container=container, + ) + + _log_command(engine.url, _cmd) + + return await asyncio.create_subprocess_exec( + *_cmd, + stdout=stdout, + stderr=asyncio.subprocess.PIPE, + ) + + +def dump_mariadb(engine: Engine, dumpfile: Path, *args, **kwargs): + task = _dump_mariadb_to_file(engine, dumpfile, *args, **kwargs) + asyncio.run(task) + + +async def _dump_mariadb_to_file(engine: Engine, dumpfile: Path, *args, **kwargs): + proc = await _dump_mariadb(engine, *args, **kwargs) + # Open dump file as an async stream + async with aiofiles.open(dumpfile, mode="wb") as dest: + await asyncio.gather( + asyncio.create_task(print_stream_progress(proc.stdout, dest)), + asyncio.create_task(print_stdout(proc.stderr)), + ) + + +def copy_mariadb_database(engine: Engine, new_engine: Engine, *args, **kwargs): + """Copy a MariaDB database to a new database in the same cluster""" + task = _copy_mariadb_database(engine, new_engine, *args, **kwargs) + asyncio.run(task) + + +async def _copy_mariadb_database(engine: Engine, new_engine: Engine, *args, **kwargs): + """Copy a MariaDB database to a new database in the same cluster""" + container = kwargs.pop("container", "mariadb:10.10") + + overwrite = kwargs.pop("overwrite", False) + create = kwargs.pop("create", True) + if database_exists(new_engine.url) and not overwrite: + console.print( + f"Database [bold underline]{new_engine.url.database}[/] already exists. Use --overwrite to overwrite." + ) + return + + dump = await _dump_mariadb(engine, *args, container=container) + restore = await _restore_mariadb( + new_engine, overwrite=overwrite, create=create, container=container + ) + + return await asyncio.gather( + asyncio.create_task( + print_stream_progress(dump.stdout, restore.stdin, prefix="Copied") + ), + asyncio.create_task(print_stdout(dump.stderr)), + asyncio.create_task(print_stdout(restore.stderr)), + ) + + +@contextmanager +def _tempfile(suffix: str = ""): + pth = Path("/tmp/sql-dump.sql") + if pth.is_file(): + pth.unlink() + yield pth diff --git a/cli/macrostrat/cli/database/mariadb/utils.py b/cli/macrostrat/cli/database/mariadb/utils.py new file mode 100644 index 00000000..ba089fa0 --- /dev/null +++ b/cli/macrostrat/cli/database/mariadb/utils.py @@ -0,0 +1,40 @@ +from sqlalchemy.engine.url import URL, make_url +from sqlalchemy.engine import create_engine +from enum import Enum + + +class ParameterStyle(Enum): + MariaDB = "mariadb" + MySQLDump = "mysqldump" + + +def build_connection_args( + url: URL, style: ParameterStyle = ParameterStyle.MariaDB +) -> [str]: + """Build MariaDB connection arguments from a SQLAlchemy URL.""" + + args = [ + "-h", + url.host, + "-u", + url.username, + ] + + if url.port: + args.extend(["-P" + str(url.port)]) + if url.password: + args.extend(["-p" + str(url.password)]) + + args.append(url.database) + + return args + + +def mariadb_engine(database: str = None): + from macrostrat.core.config import mysql_database + + _database: URL = make_url(mysql_database) + _database = _database.set(drivername="mysql+pymysql") + if database is not None: + _database = _database.set(database=database) + return create_engine(_database) diff --git a/cli/macrostrat/cli/database/utils.py b/cli/macrostrat/cli/database/utils.py new file mode 100644 index 00000000..3a21c81b --- /dev/null +++ b/cli/macrostrat/cli/database/utils.py @@ -0,0 +1,152 @@ +from ._legacy import get_db +from sqlalchemy.engine import create_engine +from sqlalchemy.engine.url import URL, make_url +from macrostrat.core.config import settings +from sqlalchemy.engine import Engine +from macrostrat.database.utils import run_sql, run_query +from psycopg2.sql import Identifier +from contextlib import contextmanager +from uuid import uuid4 +from typing import Optional + + +def engine_for_db_name(name: str | None): + engine = get_db().engine + if name is None: + return engine + url = engine.url.set(database=name) + return create_engine(url) + + +def docker_internal_url(url: URL | str) -> URL: + url = make_url(url) + if url.host == "localhost": + docker_localhost = getattr(settings, "docker_localhost", "localhost") + url = url.set(host=docker_localhost) + return url + + +@contextmanager +def pg_temp_user( + pg_engine: Engine, username: str, *, password: str = None, overwrite: bool = False +): + """Create a temporary login user for a PostgreSQL database with a limited set of permissions.""" + # Check whether the user already exists + exists = has_user(pg_engine, username) + if exists: + if overwrite: + drop_user( + pg_engine, + username, + owned_by=OwnedByPolicy.Reassign, + ) + else: + raise ValueError(f"User {username} already exists") + + if password is None: + password = str(uuid4().hex) + + run_sql( + pg_engine, + "CREATE USER {username} WITH PASSWORD :password", + dict(username=Identifier(username), password=password), + ) + + # Create a new database engine that uses the new user + url = pg_engine.url.set(username=username).set(password=password) + + try: + temp_engine = create_engine(url) + yield temp_engine + temp_engine.dispose() + finally: + # Clean up + drop_user( + pg_engine, + username, + owned_by=OwnedByPolicy.Reassign, + ) + + +from enum import Enum +from warnings import warn + + +class OwnedByPolicy(Enum): + Reassign = "reassign" + Drop = "drop" + Restrict = "restrict" + + +def drop_user( + engine: Engine, + username: str, + *, + owned_by: Optional[OwnedByPolicy] = OwnedByPolicy.Restrict, + allow_privilege_escalation: bool = True, +): + params = dict(username=Identifier(username)) + if owned_by == OwnedByPolicy.Reassign: + # Check for privilege escalation + reassign_privileges( + engine, username, allow_privilege_escalation=allow_privilege_escalation + ) + if owned_by in (OwnedByPolicy.Drop, OwnedByPolicy.Reassign): + # Drop all objects owned by the user (this actually drops permissions). + # It is hard to drop all objects owned by a user without using this sort + # of intense approach. + run_sql(engine, "DROP OWNED BY {username}", params) + + run_sql( + engine, + "DROP USER {username}", + params, + ) + + +def has_user(engine: Engine, username: str) -> bool: + """Check if a database role exists in a PostgreSQL database.""" + return ( + run_query( + engine, + "SELECT 1 FROM pg_roles WHERE rolname = :username", + dict(username=username), + ).scalar() + is not None + ) + + +def is_superuser(engine: Engine, username: str) -> bool: + return run_query( + engine, + "select usesuper from pg_user where usename = :username", + dict(username=username), + ).scalar() + + +def reassign_privileges( + engine: Engine, + from_user: str, + to_user: str = None, + *, + allow_privilege_escalation: bool = True, +): + """Reassign all objects owned by one user to another user, reporting + privilege escalation that may not be desired.""" + + if to_user is None: + to_user = engine.url.username + # Check for privilege escalation + if not is_superuser(engine, from_user) and is_superuser( + engine, engine.url.username + ): + warning = "Privilege escalation to superuser may not be desired." + if not allow_privilege_escalation: + raise ValueError(warning) + warn(warning) + + run_sql( + engine, + "REASSIGN OWNED BY {from_user} TO {to_user}", + dict(from_user=Identifier(from_user), to_user=Identifier(to_user)), + ) diff --git a/cli/poetry.lock b/cli/poetry.lock index 66b5421f..f0d56473 100644 --- a/cli/poetry.lock +++ b/cli/poetry.lock @@ -607,20 +607,22 @@ typer = ">=0.9.0,<0.10.0" [[package]] name = "macrostrat-database" -version = "3.1.2" +version = "3.3.1" description = "A SQLAlchemy-based database toolkit." optional = false -python-versions = ">=3.8,<4.0" +python-versions = "<4.0,>=3.8" files = [ - {file = "macrostrat_database-3.1.2-py3-none-any.whl", hash = "sha256:481ef7d7696c5e7cb3d537ca4e178eba97423b44dddb01af5bc2729eb89c90a4"}, - {file = "macrostrat_database-3.1.2.tar.gz", hash = "sha256:92d9e61c24428dcd485fa174e731e2da181de7b0e02c43027c82bf48e483ca21"}, + {file = "macrostrat_database-3.3.1-py3-none-any.whl", hash = "sha256:3a7adb0d3b3dc1f1e1c84a2b764bfe210f0c850d3c93e0a2256c8fd379875b75"}, + {file = "macrostrat_database-3.3.1.tar.gz", hash = "sha256:51ccc801fc5884001d6c7393ddf595688ffdc49bde0252cd65f9027619a1c51e"}, ] [package.dependencies] +aiofiles = ">=23.2.1,<24.0.0" click = ">=8.1.3,<9.0.0" GeoAlchemy2 = ">=0.14.0,<0.15.0" "macrostrat.utils" = ">=1.0.0,<2.0.0" psycopg2-binary = ">=2.9.6,<3.0.0" +rich = ">=13.7.1,<14.0.0" SQLAlchemy = ">=2.0.18,<3.0.0" SQLAlchemy-Utils = ">=0.41.1,<0.42.0" sqlparse = ">=0.4.4,<0.5.0" @@ -1310,13 +1312,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "13.5.2" +version = "13.7.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, - {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, + {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, + {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, ] [package.dependencies] @@ -1840,4 +1842,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "2492acb363222b18285ed823951258f67d09e3a8fcb52e2308ca13ef2ddfb786" +content-hash = "f1fcb36b9257e3ad57705ae57e71a851a3cc26acb01eeca72983cee80a184f31" diff --git a/cli/pyproject.toml b/cli/pyproject.toml index 7116b608..f0d29f35 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -21,7 +21,7 @@ dynaconf = "^3.1.12" geopandas = "^0.14.1" ipython = "^8.5.0" "macrostrat.app-frame" = "^1.2.4" -"macrostrat.database" = "^3.1.1" +"macrostrat.database" = "^3.3.1" numpy = "^1.23.4" psycopg2-binary = "^2.9.4" pyproj = "^3.4.0" diff --git a/core/macrostrat/core/config.py b/core/macrostrat/core/config.py index f4091e87..970f6ffd 100644 --- a/core/macrostrat/core/config.py +++ b/core/macrostrat/core/config.py @@ -4,6 +4,7 @@ from dynaconf import Dynaconf, Validator from sqlalchemy.engine import make_url from toml import load as load_toml +from sqlalchemy.engine.url import URL from .utils import find_macrostrat_config, is_pg_url @@ -54,6 +55,12 @@ def all_environments(self): docker_localhost = getattr(settings, "docker_localhost", "localhost") PG_DATABASE_DOCKER = PG_DATABASE.replace("localhost", docker_localhost) +mysql_database = getattr(settings, "mysql_database", None) +if mysql_database is not None: + mysql_database: URL = make_url(mysql_database).set(drivername="mysql+pymysql") + # TODO: handle this more intelligently + + if elevation_database := getattr(settings, "elevation_database", None): environ["ELEVATION_DATABASE_URL"] = elevation_database diff --git a/core/macrostrat/core/exc.py b/core/macrostrat/core/exc.py index 0b66a44e..d6132e32 100644 --- a/core/macrostrat/core/exc.py +++ b/core/macrostrat/core/exc.py @@ -30,7 +30,7 @@ def setup_exception_handling(app: Typer): def wrapped_app(): try: app() - except MacrostratError as error: + except ApplicationError as error: rendered = Padding(error.render(), (1, 2)) err_console.print(rendered) exit(1) diff --git a/py-root/pyproject.toml b/py-root/pyproject.toml index 3bd950f9..cb081ede 100644 --- a/py-root/pyproject.toml +++ b/py-root/pyproject.toml @@ -12,6 +12,7 @@ python = "^3.11" "macrostrat.map_integration" = { path = "../map-integration", develop = true } "criticalmaas.ta1_geopackage" = "^0.2.0" macrostrat-utils = "^1.2.1" +macrostrat-package-tools = "^1.0.0" [tool.poetry.dev-dependencies] "macrostrat.package_tools" = "^1.0.0"