From 82d124e270fba4de98674b3f6525a85620bea8ae Mon Sep 17 00:00:00 2001 From: havok2063 Date: Fri, 20 Sep 2024 18:19:07 -0400 Subject: [PATCH 1/3] updating vizdb metadata and sql for db changes --- schema/sdss5db/metadata/vizdb.json | 54 ++++++++++++++++++++++++++++++ schema/sdss5db/vizdb/vizdb.sql | 17 +++++----- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/schema/sdss5db/metadata/vizdb.json b/schema/sdss5db/metadata/vizdb.json index 215e275d..e5af6a85 100644 --- a/schema/sdss5db/metadata/vizdb.json +++ b/schema/sdss5db/metadata/vizdb.json @@ -54,6 +54,15 @@ "description": "The common, unifiying, SDSS ID of an on-sky object", "unit": "None" }, + { + "schema": "vizdb", + "table_name": "sdss_id_stacked", + "column_name": "last_updated", + "display_name": "Last Updated", + "sql_type": "date", + "description": "The date when the row was last updated", + "unit": "None" + }, { "schema": "vizdb", "table_name": "sdss_id_flat", @@ -135,6 +144,15 @@ "description": "Primary Key of the sdss_id_flat table", "unit": "None" }, + { + "schema": "vizdb", + "table_name": "sdss_id_flat", + "column_name": "rank", + "display_name": "Rank", + "sql_type": "integer", + "description": "Ranking of when catalogid paired with multiple sdss_ids. Lowest sdss_id is the preferred match, with rank 1", + "unit": "None" + }, { "schema": "vizdb", "table_name": "db_metadata", @@ -244,6 +262,15 @@ "description": "Flag indicating if the sdss_id has been reduced by the SDSS-V apogee pipeline", "unit": "None" }, + { + "schema": "vizdb", + "table_name": "sdssid_to_pipes", + "column_name": "in_bsv", + "display_name": "In Astra-Boss", + "sql_type": "boolean", + "description": "Flag indicating if the sdss_id has been reduced by the boss componenent of the Astra pipeline", + "unit": "None" + }, { "schema": "vizdb", "table_name": "sdssid_to_pipes", @@ -262,6 +289,33 @@ "description": "Flag indicating if the sdss_id target has been observed or not, either in sdss5 or dr17 release data.", "unit": "None" }, + { + "schema": "vizdb", + "table_name": "sdssid_to_pipes", + "column_name": "release", + "display_name": "Release", + "sql_type": "text", + "description": "the Astra release field indicating source reduction from sdss5 or dr17", + "unit": "None" + }, + { + "schema": "vizdb", + "table_name": "sdssid_to_pipes", + "column_name": "obs", + "display_name": "Observatory", + "sql_type": "text", + "description": "the observatory of the observation, either APO or LCO", + "unit": "None" + }, + { + "schema": "vizdb", + "table_name": "sdssid_to_pipes", + "column_name": "mjd", + "display_name": "MJD", + "sql_type": "integer", + "description": "the MJD when the target was reduced", + "unit": "None" + }, { "schema": "vizdb", "table_name": "releases", diff --git a/schema/sdss5db/vizdb/vizdb.sql b/schema/sdss5db/vizdb/vizdb.sql index e27834a4..f328a70a 100644 --- a/schema/sdss5db/vizdb/vizdb.sql +++ b/schema/sdss5db/vizdb/vizdb.sql @@ -39,23 +39,22 @@ CREATE MATERIALIZED VIEW vizdb.sdssid_to_pipes AS SELECT row_number() over(order by s.sdss_id) as pk, s.sdss_id, (b.sdss_id IS NOT NULL) AS in_boss, (v.star_pk IS NOT NULL) AS in_apogee, + (o.source_pk IS NOT NULL) AS in_bvs, (a.sdss_id IS NOT NULL) AS in_astra, - (b.sdss_id IS NOT NULL OR v.star_pk IS NOT NULL OR a.sdss_id IS NOT NULL) AS has_been_observed - -- b.id as boss_spectrum_pk, - -- v.star_pk as apogee_star_pk, - -- v.visit_pk as apogee_visit_pk, - -- a.pk as astra_source_pk, - -- v.pk as astra_apovisit_spec_pk, - -- o.pk as astra_bossvisit_spec_pk + (b.sdss_id IS NOT NULL OR v.star_pk IS NOT NULL OR a.sdss_id IS NOT NULL) AS has_been_observed, + case when b.sdss_id IS NOT NULL then 'sdss5' when v.source_pk IS NOT NULL then v.release when o.source_pk is NOT NULL then o.release end as release, + case when b.sdss_id IS NOT NULL then lower(b.obs) when v.source_pk IS NOT NULL then substring(v.telescope,0,4) when o.source_pk IS NOT NULL then substring(o.telescope,0,4) end as obs, + case when b.sdss_id IS NOT NULL then b.mjd when v.source_pk IS NOT NULL then v.mjd when o.source_pk IS NOT NULL then o.mjd end as mjd FROM vizdb.sdss_id_stacked AS s LEFT JOIN boss_drp.boss_spectrum AS b ON s.sdss_id = b.sdss_id LEFT JOIN astra_050.source AS a ON s.sdss_id = a.sdss_id LEFT JOIN astra_050.apogee_visit_spectrum as v on v.source_pk=a.pk ---LEFT JOIN astra_050.boss_visit_spectrum as o on o.source_pk=a.pk +LEFT JOIN astra_050.boss_visit_spectrum as o on o.source_pk=a.pk WITH DATA; CREATE UNIQUE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(pk); CREATE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(sdss_id); +CREATE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(mjd); -- Refresh the views with the following commands: @@ -103,4 +102,6 @@ CREATE INDEX CONCURRENTLY ON vizdb.releases USING BTREE(release); -- GRANT permissions GRANT USAGE ON SCHEMA vizdb TO sdss; +GRANT SELECT ON vizdb.sdss_id_stacked TO sdss; +GRANT SELECT ON vizdb.sdss_id_flat TO sdss; GRANT SELECT ON vizdb.sdssid_to_pipes TO sdss; From 0f204acf348dce29676a147a5c2d4449bdef49e1 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Mon, 23 Sep 2024 15:49:15 -0400 Subject: [PATCH 2/3] updating generate releases code --- schema/sdss5db/vizdb/gen_versions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schema/sdss5db/vizdb/gen_versions.py b/schema/sdss5db/vizdb/gen_versions.py index ca2c901b..7c659a05 100644 --- a/schema/sdss5db/vizdb/gen_versions.py +++ b/schema/sdss5db/vizdb/gen_versions.py @@ -45,6 +45,7 @@ def create_df() -> list[dict]: df = fix_list(df, 'run2d') df = fix_list(df, 'run1d') df = fix_list(df, 'apred_vers') + df = fix_list(df, 'v_astra') # drop legacy rows df = df.set_index('release', drop=False) @@ -54,8 +55,8 @@ def create_df() -> list[dict]: # add mjd cutoffs # todo - move this to the datamodel df.loc['DR18', 'mjd_cutoff_apo'] = 59392 - df.loc['DR19', 'mjd_cutoff_apo'] = 60280 - df.loc['DR19', 'mjd_cutoff_lco'] = 60280 + df.loc['DR19', 'mjd_cutoff_apo'] = 60130 + df.loc['DR19', 'mjd_cutoff_lco'] = 60130 df.loc['IPL3', 'mjd_cutoff_apo'] = 60130 df.loc['IPL1', 'mjd_cutoff_apo'] = 59765 From e98e5f06105b9398882ed66d9ab615312e6a0c89 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Mon, 23 Sep 2024 16:07:18 -0400 Subject: [PATCH 3/3] updating orms for schema changes --- python/sdssdb/peewee/sdss5db/vizdb.py | 8 +++++++- python/sdssdb/sqlalchemy/sdss5db/vizdb.py | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/sdssdb/peewee/sdss5db/vizdb.py b/python/sdssdb/peewee/sdss5db/vizdb.py index 9069352c..9e0affab 100644 --- a/python/sdssdb/peewee/sdss5db/vizdb.py +++ b/python/sdssdb/peewee/sdss5db/vizdb.py @@ -8,7 +8,7 @@ # Database: sdss5db # Peewee version: 3.13.2 -from peewee import (AutoField, BigIntegerField, BooleanField, +from peewee import (AutoField, BigIntegerField, BooleanField, DateField, DoubleField, IntegerField, SmallIntegerField, TextField) from playhouse.postgres_ext import ArrayField @@ -34,6 +34,7 @@ class SDSSidFlat(VizBase): pk = AutoField() ra_catalogid = DoubleField(null=True) dec_catalogid = DoubleField(null=True) + rank = IntegerField(null=True) class Meta: table_name = 'sdss_id_flat' @@ -48,6 +49,7 @@ class SDSSidStacked(VizBase): ra_sdss_id = DoubleField(null=True) dec_sdss_id = DoubleField(null=True) sdss_id = BigIntegerField(null=False, primary_key=True) + last_updated = DateField(null=True) class Meta: table_name = 'sdss_id_stacked' @@ -60,8 +62,12 @@ class SDSSidToPipes(VizBase): sdss_id = BigIntegerField(null=False) in_boss = BooleanField(null=False) in_apogee = BooleanField(null=False) + in_bvs = BooleanField(null=False) in_astra = BooleanField(null=False) has_been_observed = BooleanField(null=False) + release = TextField(null=True) + obs = TextField(null=True) + mjd = IntegerField(null=True) class Meta: table_name = 'sdssid_to_pipes' diff --git a/python/sdssdb/sqlalchemy/sdss5db/vizdb.py b/python/sdssdb/sqlalchemy/sdss5db/vizdb.py index eb6b2c4a..beeb0016 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/vizdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/vizdb.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr -from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, Float, Integer, +from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, Date, Float, Integer, SmallInteger, Text, text) from sdssdb.sqlalchemy.sdss5db import SDSS5dbBase, database @@ -53,6 +53,8 @@ class SDSSidFlat(Base): pk = Column('pk', BigInteger, primary_key=True) ra_catalogid = Column('ra_catalogid', Float(53)) dec_catalogid = Column('dec_catalogid', Float(53)) + rank = Column('rank', Integer) + class SDSSidStacked(Base): @@ -65,6 +67,7 @@ class SDSSidStacked(Base): ra_sdss_id = Column('ra_sdss_id', Float(53)) dec_sdss_id = Column('dec_sdss_id', Float(53)) sdss_id = Column('sdss_id', BigInteger, primary_key=True) + last_updated = Column('last_updated', Date) class SDSSidToPipes(Base): @@ -75,8 +78,12 @@ class SDSSidToPipes(Base): sdss_id = Column('sdss_id', BigInteger) in_boss = Column('in_boss', Boolean) in_apogee = Column('in_apogee', Boolean) + in_bvs = Column('in_bvs', Boolean) in_astra = Column('in_astra', Boolean) has_been_observed = Column('has_been_observed', Boolean) + mjd = Column('release', Text) + mjd = Column('obs', Text) + mjd = Column('mjd', Integer) class Releases(Base):