Merge branch 'refactor_ingest_spectral_types_function' of github.com:…

…Exu-112/SIMPLE-db into Roth24-sptypes

documentation/SpectralTypes.md

@@ -10,7 +10,7 @@ Columns marked with an asterisk (*) may not be empty.
 | spectral_type_string | Spectral type string |  | String(20)  |   |
 | *spectral_type_code | Numeric code corresponding to spectral type |  | Float  | primary  |
 | spectral_type_error | Uncertainty of spectral type |  | Float  |   |
-| regime | Regime for spectral type value |  |  | primary and foreign:Regimes.regime |
+| *regime | Regime for spectral type value |  |  | primary and foreign:Regimes.regime |
 | adopted    | Flag indicating if this is the adopted measurement |  | Boolean  |   |
 | photometric    | Flag indicating if this is a photometric spectral type |  | Boolean  |   |
 | comments  | Free form comments |   | String(1000) |   |
@@ -21,4 +21,6 @@ Spectral Type Codes:
  - 69 = M9
  - 70 = L0 
  - 80 = T0
- - 90 = Y0
+ - 90 = Y0
+
+ Regime is required however, regime = "unkwown" can be used when the regime is unknown. 

simple/schema.py

@@ -306,7 +306,6 @@ class SpectralTypes(Base):
     regime = Column(
         String(30),
         ForeignKey("Regimes.regime", ondelete="cascade", onupdate="cascade"),
-        nullable=True,
         primary_key=True,
     )
     adopted = Column(Boolean)
@@ -318,7 +317,7 @@ class SpectralTypes(Base):
         primary_key=True,
     )
 
-    @validates("source", "spectral_type_code", "reference")
+    @validates("source", "spectral_type_code", "regime", "reference")
     def validate_required(self, key, value):
         if value is None:
             raise ValueError(f"Value required for {key}")

simple/utils/spectral_types.py

@@ -17,6 +17,7 @@
 ]
 
 logger = logging.getLogger("SIMPLE")
+logger.setLevel(logging.INFO)  # set default log level to INFO
 
 
 def ingest_spectral_type(
@@ -72,6 +73,26 @@ def ingest_spectral_type(
         db.query(db.SpectralTypes).filter(db.SpectralTypes.c.source == db_name).table()
     )
 
+    # Check for duplicates
+    duplicate_check = (
+        db.query(db.SpectralTypes.c.source)
+        .filter(
+            and_(
+                db.SpectralTypes.c.source == db_name,
+                db.SpectralTypes.c.regime == regime,
+                db.SpectralTypes.c.reference == reference,
+            )
+        )
+        .count()
+    )
+    if duplicate_check > 0:
+        msg = f"Spectral type already in the database: {db_name}, {regime}, {reference}"
+        if raise_error:
+            logger.error(msg)
+            raise AstroDBError(msg)
+        else:
+            logger.warning(msg)
+
     # set adopted flag
     if source_spt_data is None or len(source_spt_data) == 0:
         adopted = True
@@ -99,9 +120,12 @@ def ingest_spectral_type(
                     adopted = True
                 logger.debug(f"The new spectral type's adopted flag is:, {adopted}")
     else:
-        msg = "Unexpected state"
-        logger.error(msg)
-        raise RuntimeError
+        msg = f"Unexpected state while ingesting {source}"
+        if raise_error:
+            logger.error(msg)
+            raise RuntimeError
+        else:
+            logger.warning(msg)
 
     if spectral_type_code is None:
         spectral_type_code = convert_spt_string_to_code(spectral_type_string)
@@ -121,21 +145,6 @@ def ingest_spectral_type(
         "reference": reference,
     }
 
-    check = (
-        db.query(db.SpectralTypes.c.source)
-        .filter(
-            and_(
-                db.SpectralTypes.c.source == db_name,
-                db.SpectralTypes.c.regime == regime,
-                db.SpectralTypes.c.reference == reference,
-            )
-        )
-        .count()
-    )
-    if check == 1:
-        msg = f"Spectral type for {db_name} already in the database"
-        raise AstroDBError(msg)
-
     logger.debug(f"Trying to insert {spt_data} into Spectral Types table ")
 
     try:
@@ -181,17 +190,20 @@ def ingest_spectral_type(
             == 0
         ):
             msg = f"The publication does not exist in the database: {reference}"
-            msg1 = "Add it with ingest_publication function."
+            msg1 = "Add it with astrodb_utils.ingest_publication function."
             logger.debug(msg + msg1)
-            raise AstroDBError(msg)
-        elif "NOT NULL constraint failed: SpectralTypes.regime" in str(e):
-            msg = f"The regime was not provided for {source}"
-            logger.error(msg)
-            raise AstroDBError(msg)
+            if raise_error:
+                logger.error(msg)
+                raise AstroDBError(msg + msg1)
+            else:
+                logger.warning(msg)
         else:
             msg = f"Spectral type ingest failed with error {e}\n"
-            logger.error(msg)
-            raise AstroDBError(msg)
+            if raise_error:
+                logger.error(msg)
+                raise AstroDBError(msg)
+            else:
+                logger.warning(msg)
 
 
 def convert_spt_string_to_code(spectral_type_string):

tests/test_utils.py

@@ -146,9 +146,7 @@ def test_ingest_spectral_type_errors(temp_db):
             reference=spt_data4["reference"],
             regime=spt_data4["regime"],
         )
-    assert "Spectral type for Fake 1 already in the database" in str(
-        error_message.value
-    )
+    assert "Spectral type already in the database" in str(error_message.value)
     # assert "The publication does not exist in the database" in str(error_message.value)