Updated working script and tests

scripts/ingests/BYW_SpT2024.py

@@ -1,19 +1,26 @@
 from astrodb_utils import load_astrodb
+from astrodb_utils import ingest_publication
 from simple.schema import *
+from simple.schema import REFERENCE_TABLES
+from astropy.io import ascii
+from simple.utils.spectral_types import ingest_spectral_type
+import logging
 
+logger = logging.getLogger("SIMPLE")
+logger.setLevel(logging.INFO)
 SAVE_DB = False  # save the data files in addition to modifying the .db file
 RECREATE_DB = True  # recreates the .db file from the data files
 # LOAD THE DATABASE
-db = load_astrodb("SIMPLE.db", recreatedb=RECREATE_DB)
-
+db = load_astrodb("SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES)
+ingest_publication(db, doi='10.3847/1538-3881/ad324e' )
 # Load Google sheet
-sheet_id = "1JFa8F4Ngzp3qAW8NOBurkz4bMKo9zXYeF6N1vMtqDZs"
-link = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv"
+
+link = 'scripts/ingests/Austin-BYW-Benchmark-SpT.csv'
 
 # read the csv data into an astropy table
 # ascii.read attempts to read data from local files rather from URLs so using a library like requests helps get data and create object that can be passed to ascii.read
 byw_table = ascii.read(
-    link,
+    link, #change this to the path for the csv file
     format="csv",
     data_start=1,
     header_start=0,
@@ -22,32 +29,29 @@
     delimiter=",",
 )
 
-# print result astropy table
-print(byw_table.info)
-
-# Loop through each row in byw table and print data: source name ra, dec.
-def ingest_all_source_spt(db):
-    for row in byw_table[1:90]:  # skip the header row - [1:10]runs only first 10 rows
-        # Print byw source information
-        print("BYW Source SpT Information:")
+for row in byw_table:  # skip the header row - [1:10]runs only first 10 rows
+    if row["photometric"] == 'True': 
+        photometric = True
+    else:
+        photometric = False
 
+    preexisting = ['CWISE J183207.94-540943.3',
+                   'SDSS J134403.83+083950.9']
+    if row["Source"] in preexisting:
+        continue
 
-        for col_name in row.colnames:
-            print(f"{col_name}: {row[col_name]}")
-
-        ingest_spectral_types(
+    ingest_spectral_type(
             db,
             source=row["Source"],
             spectral_type_string=row["spectral_type_string"],
-            spectral_type_code=row["spectral_type_code"],
             spectral_type_error=row["spectral_type_error"],
             regime=row["regime"],
-            adopted=row["adopted"],
             comments=row["comments"],
-            reference=row["Reference"]
-            raise_error=True,
-            search_db=True,
-        )
-
-        # Add a separator between rows for better readability
-        print("-" * 20)    
+            photometric=photometric,
+            reference=row["Reference"],
+            raise_error=False,
+        ) 
+
+    # WRITE THE JSON FILES
+if SAVE_DB:
+    db.save_database(directory="data/")

tests/test_data.py

@@ -324,7 +324,7 @@ def test_spectral_types(db):
 
     regime = "nir"
     t = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.regime == regime).astropy()
-    assert len(t) == 2359, f"found {len(t)} spectral types in the {regime} regime"
+    assert len(t) == 2446, f"found {len(t)} spectral types in the {regime} regime"
 
     regime = "nir_UCD"
     t = db.query(db.SpectralTypes).filter(db.SpectralTypes.c.regime == regime).astropy()
@@ -353,7 +353,7 @@ def test_spectral_types(db):
         )
         .astropy()
     )
-    assert len(m_dwarfs) == 843, f"found {len(t)} M spectral types"
+    assert len(m_dwarfs) == 861, f"found {len(t)} M spectral types"
 
     l_dwarfs = (
         db.query(db.SpectralTypes)
@@ -365,7 +365,7 @@ def test_spectral_types(db):
         )
         .astropy()
     )
-    assert len(l_dwarfs) == 1963, f"found {len(l_dwarfs)} L spectral types"
+    assert len(l_dwarfs) == 2011, f"found {len(l_dwarfs)} L spectral types"
 
     t_dwarfs = (
         db.query(db.SpectralTypes)
@@ -377,7 +377,7 @@ def test_spectral_types(db):
         )
         .astropy()
     )
-    assert len(t_dwarfs) == 998, f"found {len(t_dwarfs)} T spectral types"
+    assert len(t_dwarfs) == 1019, f"found {len(t_dwarfs)} T spectral types"
 
     y_dwarfs = (
         db.query(db.SpectralTypes)