Skip to content

Commit

Permalink
Iterating on error message handling for ingest_spectrum
Browse files Browse the repository at this point in the history
  • Loading branch information
@dr-rodriguez
dr-rodriguez committed Jul 9, 2024
1 parent b5a44c8 commit 578d231
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 64 deletions.
20 changes: 10 additions & 10 deletions simple/utils/spectra.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,13 @@ def ingest_spectrum(
mode=mode,
)
if len(matches) > 0:
msg = f"Skipping suspected duplicate measurement\n{source}\n"
msg = f"Skipping suspected duplicate measurement: {source}\n"
msg2 = f"{matches}" f"{instrument, mode, obs_date, reference, spectrum} \n"
logger.warning(msg)
logger.debug(msg2)
flags["message"] = msg
if raise_error:
raise AstroDBError
raise AstroDBError(msg)
else:
return flags

Expand Down Expand Up @@ -194,25 +194,25 @@ def ingest_spectrum(
flags["added"] = True
logger.info(f"Added {source} : \n" f"{row_data}")
except sqlalchemy.exc.IntegrityError as e:
msg = "Integrity Error:" f"{source} \n" f"{row_data}"
logger.error(msg + str(e) + f" \n {row_data}")
msg = f"Integrity Error: {source} \n {e}"
logger.error(msg + f" \n {row_data}")
flags["message"] = msg
if raise_error:
raise AstroDBError(msg)
else:
return flags
except sqlite3.IntegrityError as e:
msg = "Integrity Error: " f"{source} \n" f"{row_data}"
logger.error(msg + str(e))
msg = f"Integrity Error: {source} \n {e}"
logger.error(msg)
flags["message"] = msg
if raise_error:
raise AstroDBError(msg)
else:
return flags
except Exception as e:
msg = (
f"Spectrum for {source} could not be added to the database"
f"for unexpected reason: \n {row_data} \n error: {e}"
f"Spectrum for {source} could not be added to the database "
f"for unexpected reason: {e}"
)
logger.error(msg)
flags["message"] = msg
Expand Down Expand Up @@ -308,7 +308,7 @@ def spectrum_plottable(spectrum_path, raise_error=True, show_plot=False):
return False
except u.UnitConversionError as e:
msg = (
f"{str(e)} \n"
f"{e} \n"
f"Skipping {spectrum_path}: unable to convert spectral axis to microns"
)
if raise_error:
Expand All @@ -318,7 +318,7 @@ def spectrum_plottable(spectrum_path, raise_error=True, show_plot=False):
logger.warning(msg)
return False
except ValueError as e:
msg = f"{str(e)} \nSkipping {spectrum_path}: Value error"
msg = f"{e} \nSkipping {spectrum_path}: Value error"
if raise_error:
logger.error(msg)
raise AstroDBError(msg)
Expand Down
123 changes: 69 additions & 54 deletions tests/test_spectra_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
)


@pytest.mark.filterwarnings("ignore::UserWarning")
@pytest.mark.filterwarnings("ignore", message=".*not FITS standard.*")
@pytest.mark.filterwarnings(
"ignore", message=".*Note: astropy.io.fits uses zero-based indexing.*"
)
Expand All @@ -30,47 +28,55 @@ def test_ingest_spectrum_errors(temp_db):

# A lot of the tests fail because they were checking very specific parts of ingest_spectrum

# Ingesting a spectrum with missing regime
spectrum = "https://bdnyc.s3.amazonaws.com/tests/U10176.fits"
with pytest.raises(AstroDBError) as error_message:
ingest_spectrum(temp_db, source="apple", spectrum=spectrum)
# assert "Regime is required" in str(error_message.value)
# Commented out the error_message assertions since we have more generic error messages with this
_ = ingest_spectrum(temp_db, source="apple", spectrum=spectrum)
assert "Value required for regime" in str(error_message.value)

result = ingest_spectrum(
temp_db, source="apple", spectrum=spectrum, raise_error=False
)
assert result["added"] is False
assert result["skipped"] is True
assert "Value required for regime" in result["message"]


# Ingesting with missing reference
with pytest.raises(AstroDBError) as error_message:
ingest_spectrum(
temp_db,
source="apple",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
regime="nir",
spectrum=spectrum,
)
_ = ingest_spectrum(
temp_db,
source="apple",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
regime="nir",
spectrum=spectrum,
obs_date="2020-01-01",
)
assert "NOT NULL constraint failed: Spectra.reference" in str(error_message.value)
# assert "Reference is required" in str(error_message.value)

result = ingest_spectrum(
temp_db, source="apple", regime="nir", spectrum=spectrum, raise_error=False
temp_db, source="apple", regime="nir", spectrum=spectrum, raise_error=False, obs_date="2020-01-01"
)
assert result["added"] is False
assert result["skipped"] is True
assert "NOT NULL constraint failed: Spectra.reference" in result["message"]


# Ingesting with invalid reference (does not already exist)
with pytest.raises(AstroDBError) as error_message:
ingest_spectrum(
temp_db,
source="apple",
regime="nir",
spectrum=spectrum,
telescope="IRTF",
instrument="SpeX",
mode="Prism",
reference="Ref 5",
)
_ = ingest_spectrum(
temp_db,
source="apple",
regime="nir",
spectrum=spectrum,
telescope="IRTF",
instrument="SpeX",
mode="Prism",
reference="Ref 5",
obs_date="2020-01-01",
)
print(error_message)
# assert "not in Publications table" in str(error_message.value)

result = ingest_spectrum(
Expand All @@ -83,21 +89,26 @@ def test_ingest_spectrum_errors(temp_db):
mode="Prism",
reference="Ref 5",
raise_error=False,
obs_date="2020-01-01",
)
for k, v in result.items():
print(k, v)
assert result["added"] is False
assert result["skipped"] is True


# Ingesting for invalid source (not already in database)
with pytest.raises(AstroDBError) as error_message:
ingest_spectrum(
temp_db,
source="kiwi",
regime="nir",
spectrum=spectrum,
reference="Ref 1",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
)
_ = ingest_spectrum(
temp_db,
source="kiwi",
regime="nir",
spectrum=spectrum,
reference="Ref 1",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
obs_date="2020-01-01",
)
# assert "No unique source match for kiwi in the database" in str(error_message.value)

result = ingest_spectrum(
Expand All @@ -110,21 +121,25 @@ def test_ingest_spectrum_errors(temp_db):
telescope="IRTF",
instrument="SpeX",
mode="Prism",
obs_date="2020-01-01",
)
print(result)
assert result["added"] is False
assert result["skipped"] is True


# Ingesting with missing date
with pytest.raises(AstroDBError) as error_message:
ingest_spectrum(
temp_db,
source="apple",
regime="nir",
spectrum=spectrum,
reference="Ref 1",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
)
_ = ingest_spectrum(
temp_db,
source="apple",
regime="nir",
spectrum=spectrum,
reference="Ref 1",
telescope="IRTF",
instrument="SpeX",
mode="Prism",
)
assert "Invalid date received" in str(error_message.value)
# assert "missing observation date" in str(error_message.value)

result = ingest_spectrum(
Expand All @@ -139,11 +154,12 @@ def test_ingest_spectrum_errors(temp_db):
raise_error=False,
)
assert result["added"] is False
# assert result["skipped"] is False
# assert result["no_obs_date"] is True
assert "Invalid date received" in result["message"]


# Ingesting with invalid regime
with pytest.raises(AstroDBError) as error_message:
result = ingest_spectrum(
_ = ingest_spectrum(
temp_db,
source="orange",
regime="far-uv",
Expand All @@ -168,9 +184,8 @@ def test_ingest_spectrum_errors(temp_db):
mode="OG570",
raise_error=False,
)
print(result)
assert result["added"] is False
assert result["skipped"] is True


@pytest.mark.filterwarnings("ignore:Verification")
@pytest.mark.filterwarnings("ignore", message=".*Card 'AIRMASS' is not FITS standard.*")
Expand Down

0 comments on commit 578d231

Please sign in to comment.