Update vizdb orms for schema changes (#272)

* updating vizdb metadata and sql for db changes

* updating generate releases code

* updating orms for schema changes

python/sdssdb/peewee/sdss5db/vizdb.py

@@ -8,7 +8,7 @@
 # Database: sdss5db
 # Peewee version: 3.13.2
 
-from peewee import (AutoField, BigIntegerField, BooleanField,
+from peewee import (AutoField, BigIntegerField, BooleanField, DateField,
                     DoubleField, IntegerField, SmallIntegerField, TextField)
 from playhouse.postgres_ext import ArrayField
 
@@ -34,6 +34,7 @@ class SDSSidFlat(VizBase):
     pk = AutoField()
     ra_catalogid = DoubleField(null=True)
     dec_catalogid = DoubleField(null=True)
+    rank = IntegerField(null=True)
 
     class Meta:
         table_name = 'sdss_id_flat'
@@ -48,6 +49,7 @@ class SDSSidStacked(VizBase):
     ra_sdss_id = DoubleField(null=True)
     dec_sdss_id = DoubleField(null=True)
     sdss_id = BigIntegerField(null=False, primary_key=True)
+    last_updated = DateField(null=True)
 
     class Meta:
         table_name = 'sdss_id_stacked'
@@ -60,8 +62,12 @@ class SDSSidToPipes(VizBase):
     sdss_id = BigIntegerField(null=False)
     in_boss = BooleanField(null=False)
     in_apogee = BooleanField(null=False)
+    in_bvs = BooleanField(null=False)
     in_astra = BooleanField(null=False)
     has_been_observed = BooleanField(null=False)
+    release = TextField(null=True)
+    obs = TextField(null=True)
+    mjd = IntegerField(null=True)
 
     class Meta:
         table_name = 'sdssid_to_pipes'

python/sdssdb/sqlalchemy/sdss5db/vizdb.py

@@ -7,7 +7,7 @@
 
 
 from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr
-from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, Float, Integer,
+from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, Date, Float, Integer,
                         SmallInteger, Text, text)
 
 from sdssdb.sqlalchemy.sdss5db import SDSS5dbBase, database
@@ -53,6 +53,8 @@ class SDSSidFlat(Base):
     pk = Column('pk', BigInteger, primary_key=True)
     ra_catalogid = Column('ra_catalogid', Float(53))
     dec_catalogid = Column('dec_catalogid', Float(53))
+    rank = Column('rank', Integer)
+
 
 
 class SDSSidStacked(Base):
@@ -65,6 +67,7 @@ class SDSSidStacked(Base):
     ra_sdss_id = Column('ra_sdss_id', Float(53))
     dec_sdss_id = Column('dec_sdss_id', Float(53))
     sdss_id = Column('sdss_id', BigInteger, primary_key=True)
+    last_updated = Column('last_updated', Date)
 
 
 class SDSSidToPipes(Base):
@@ -75,8 +78,12 @@ class SDSSidToPipes(Base):
     sdss_id = Column('sdss_id', BigInteger)
     in_boss = Column('in_boss', Boolean)
     in_apogee = Column('in_apogee', Boolean)
+    in_bvs = Column('in_bvs', Boolean)
     in_astra = Column('in_astra', Boolean)
     has_been_observed = Column('has_been_observed', Boolean)
+    mjd = Column('release', Text)
+    mjd = Column('obs', Text)
+    mjd = Column('mjd', Integer)
 
 
 class Releases(Base):

schema/sdss5db/metadata/vizdb.json

@@ -54,6 +54,15 @@
       "description": "The common, unifiying, SDSS ID of an on-sky object",
       "unit": "None"
     },
+    {
+      "schema": "vizdb",
+      "table_name": "sdss_id_stacked",
+      "column_name": "last_updated",
+      "display_name": "Last Updated",
+      "sql_type": "date",
+      "description": "The date when the row was last updated",
+      "unit": "None"
+    },
     {
       "schema": "vizdb",
       "table_name": "sdss_id_flat",
@@ -135,6 +144,15 @@
       "description": "Primary Key of the sdss_id_flat table",
       "unit": "None"
     },
+    {
+      "schema": "vizdb",
+      "table_name": "sdss_id_flat",
+      "column_name": "rank",
+      "display_name": "Rank",
+      "sql_type": "integer",
+      "description": "Ranking of when catalogid paired with multiple sdss_ids.  Lowest sdss_id is the preferred match, with rank 1",
+      "unit": "None"
+    },
     {
       "schema": "vizdb",
       "table_name": "db_metadata",
@@ -244,6 +262,15 @@
       "description": "Flag indicating if the sdss_id has been reduced by the SDSS-V apogee pipeline",
       "unit": "None"
     },
+    {
+      "schema": "vizdb",
+      "table_name": "sdssid_to_pipes",
+      "column_name": "in_bsv",
+      "display_name": "In Astra-Boss",
+      "sql_type": "boolean",
+      "description": "Flag indicating if the sdss_id has been reduced by the boss componenent of the Astra pipeline",
+      "unit": "None"
+    },
     {
       "schema": "vizdb",
       "table_name": "sdssid_to_pipes",
@@ -262,6 +289,33 @@
       "description": "Flag indicating if the sdss_id target has been observed or not, either in sdss5 or dr17 release data.",
       "unit": "None"
     },
+    {
+      "schema": "vizdb",
+      "table_name": "sdssid_to_pipes",
+      "column_name": "release",
+      "display_name": "Release",
+      "sql_type": "text",
+      "description": "the Astra release field indicating source reduction from sdss5 or dr17",
+      "unit": "None"
+    },
+    {
+      "schema": "vizdb",
+      "table_name": "sdssid_to_pipes",
+      "column_name": "obs",
+      "display_name": "Observatory",
+      "sql_type": "text",
+      "description": "the observatory of the observation, either APO or LCO",
+      "unit": "None"
+    },
+    {
+      "schema": "vizdb",
+      "table_name": "sdssid_to_pipes",
+      "column_name": "mjd",
+      "display_name": "MJD",
+      "sql_type": "integer",
+      "description": "the MJD when the target was reduced",
+      "unit": "None"
+    },
     {
       "schema": "vizdb",
       "table_name": "releases",

schema/sdss5db/vizdb/gen_versions.py

@@ -45,6 +45,7 @@ def create_df() -> list[dict]:
     df = fix_list(df, 'run2d')
     df = fix_list(df, 'run1d')
     df = fix_list(df, 'apred_vers')
+    df = fix_list(df, 'v_astra')
 
     # drop legacy rows
     df = df.set_index('release', drop=False)
@@ -54,8 +55,8 @@ def create_df() -> list[dict]:
     # add mjd cutoffs
     # todo - move this to the datamodel
     df.loc['DR18', 'mjd_cutoff_apo'] = 59392
-    df.loc['DR19', 'mjd_cutoff_apo'] = 60280
-    df.loc['DR19', 'mjd_cutoff_lco'] = 60280
+    df.loc['DR19', 'mjd_cutoff_apo'] = 60130
+    df.loc['DR19', 'mjd_cutoff_lco'] = 60130
     df.loc['IPL3', 'mjd_cutoff_apo'] = 60130
     df.loc['IPL1', 'mjd_cutoff_apo'] = 59765
 

schema/sdss5db/vizdb/vizdb.sql

@@ -39,23 +39,22 @@ CREATE MATERIALIZED VIEW vizdb.sdssid_to_pipes AS
 SELECT row_number() over(order by s.sdss_id) as pk, s.sdss_id,
        (b.sdss_id IS NOT NULL) AS in_boss,
        (v.star_pk IS NOT NULL) AS in_apogee,
+	   (o.source_pk IS NOT NULL) AS in_bvs,
        (a.sdss_id IS NOT NULL) AS in_astra,
-       (b.sdss_id IS NOT NULL OR v.star_pk IS NOT NULL OR a.sdss_id IS NOT NULL) AS has_been_observed
-    --    b.id as boss_spectrum_pk,
-    --    v.star_pk as apogee_star_pk,
-    --    v.visit_pk as apogee_visit_pk,
-    --    a.pk as astra_source_pk,
-    --    v.pk as astra_apovisit_spec_pk,
-    --    o.pk as astra_bossvisit_spec_pk
+       (b.sdss_id IS NOT NULL OR v.star_pk IS NOT NULL OR a.sdss_id IS NOT NULL) AS has_been_observed,
+       case when b.sdss_id IS NOT NULL then 'sdss5' when v.source_pk IS NOT NULL then v.release when o.source_pk is NOT NULL then o.release end as release,
+       case when b.sdss_id IS NOT NULL then lower(b.obs) when v.source_pk IS NOT NULL then substring(v.telescope,0,4) when o.source_pk IS NOT NULL then substring(o.telescope,0,4) end as obs,
+       case when b.sdss_id IS NOT NULL then b.mjd when v.source_pk IS NOT NULL then v.mjd when o.source_pk IS NOT NULL then o.mjd end as mjd
 FROM vizdb.sdss_id_stacked AS s
 LEFT JOIN boss_drp.boss_spectrum AS b ON s.sdss_id = b.sdss_id
 LEFT JOIN astra_050.source AS a ON s.sdss_id = a.sdss_id
 LEFT JOIN astra_050.apogee_visit_spectrum as v on v.source_pk=a.pk
---LEFT JOIN astra_050.boss_visit_spectrum as o on o.source_pk=a.pk
+LEFT JOIN astra_050.boss_visit_spectrum as o on o.source_pk=a.pk
 WITH DATA;
 
 CREATE UNIQUE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(pk);
 CREATE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(sdss_id);
+CREATE INDEX CONCURRENTLY ON vizdb.sdssid_to_pipes USING BTREE(mjd);
 
 
 -- Refresh the views with the following commands:
@@ -103,4 +102,6 @@ CREATE INDEX CONCURRENTLY ON vizdb.releases USING BTREE(release);
 
 -- GRANT permissions
 GRANT USAGE ON SCHEMA vizdb TO sdss;
+GRANT SELECT ON vizdb.sdss_id_stacked TO sdss;
+GRANT SELECT ON vizdb.sdss_id_flat TO sdss;
 GRANT SELECT ON vizdb.sdssid_to_pipes TO sdss;