-
Notifications
You must be signed in to change notification settings - Fork 22
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Ingest YJHK MKO photometry from ultracool sheet #533
Merged
kelle
merged 16 commits into
SIMPLE-AstroDB:main
from
Exu-112:ultracool_MKO_photometry_ingest
Jul 15, 2024
Merged
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit
Hold shift + click to select a range
66436be
created script!
Exu-112 9f38361
use the ingest function from utils
Exu-112 0ccc09a
made script ingest names
Exu-112 5a5a57f
JSONs and tests updated
Exu-112 9114077
revert jsons
Exu-112 902893d
Update Ingest_YJHK_MKO_Photometry.py
Exu-112 064232f
fixed script
Exu-112 83453c0
json
Exu-112 e6a131a
updated reference logic
Exu-112 67a3d34
delete unused code
Exu-112 14fb178
Update references.py
Exu-112 187c2c9
reference bug fixed
Exu-112 d6c7f36
tests
Exu-112 422e421
WFAU ref and updated tests
Exu-112 5473f49
versions update
Exu-112 a8ac96a
a
Exu-112 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
208 changes: 208 additions & 0 deletions
208
scripts/ingests/ultracool_sheet/Ingest_YJHK_MKO_Photometry.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,208 @@ | ||
from astrodb_utils import load_astrodb, find_source_in_db, AstroDBError | ||
import sys | ||
|
||
sys.path.append(".") | ||
import logging | ||
from astropy.io import ascii | ||
from simple.schema import Photometry | ||
from simple.schema import REFERENCE_TABLES | ||
from math import isnan | ||
import sqlalchemy.exc | ||
from simple.utils.astrometry import ingest_parallax | ||
from scripts.ingests.ultracool_sheet.references import uc_ref_to_simple_ref | ||
|
||
|
||
logger = logging.getLogger(__name__) | ||
|
||
# Logger setup | ||
# This will stream all logger messages to the standard output and | ||
# apply formatting for that | ||
logger.propagate = False # prevents duplicated logging messages | ||
LOGFORMAT = logging.Formatter( | ||
"%(asctime)s %(levelname)s: %(message)s", datefmt="%m/%d/%Y %I:%M:%S%p" | ||
) | ||
ch = logging.StreamHandler(stream=sys.stdout) | ||
ch.setFormatter(LOGFORMAT) | ||
# To prevent duplicate handlers, only add if they haven't been set previously | ||
if len(logger.handlers) == 0: | ||
logger.addHandler(ch) | ||
logger.setLevel(logging.INFO) | ||
|
||
DB_SAVE = False | ||
RECREATE_DB = True | ||
db = load_astrodb( | ||
"SIMPLE.sqlite", recreatedb=RECREATE_DB, reference_tables=REFERENCE_TABLES | ||
) | ||
|
||
# Load Ultracool sheet | ||
doc_id = "1i98ft8g5mzPp2DNno0kcz4B9nzMxdpyz5UquAVhz-U8" | ||
sheet_id = "361525788" | ||
link = ( | ||
f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=csv&gid={sheet_id}" | ||
) | ||
|
||
# read the csv data into an astropy table | ||
uc_sheet_table = ascii.read( | ||
link, | ||
format="csv", | ||
data_start=1, | ||
header_start=0, | ||
guess=False, | ||
fast_reader=False, | ||
delimiter=",", | ||
) | ||
|
||
|
||
def ingest_Photometry( | ||
source: str = None, | ||
band: str = None, | ||
magnitude: float = None, | ||
mag_error: float = None, | ||
telescope: str = None, | ||
# instrument: str = None, | ||
comment: str = None, | ||
reference: str = None, | ||
raise_error: bool = True, | ||
): | ||
table_data = { | ||
"source": source, | ||
"band": band, | ||
"magnitude": magnitude, | ||
"magnitude_error": mag_error, | ||
"telescope": telescope, | ||
# "instrument": instrument, | ||
"comments": comment, | ||
"reference": reference, | ||
} | ||
|
||
try: | ||
pho_obj = Photometry(**table_data) | ||
with db.session as session: | ||
session.add(pho_obj) | ||
session.commit() | ||
logger.info(f" Photometry added to database: {table_data}\n") | ||
except sqlalchemy.exc.IntegrityError as e: | ||
if "UNIQUE constraint failed:" in str(e): | ||
msg = f" Photometry with same reference already exists in database: {table_data}" | ||
if raise_error: | ||
raise AstroDBError(msg) from e | ||
else: | ||
msg2 = f"SKIPPING: {source}. Photometry with same reference already exists in database." | ||
logger.warning(msg2) | ||
else: | ||
msg = f"Could not add {table_data} to database. Error: {e}" | ||
logger.warning(msg) | ||
raise AstroDBError(msg) from e | ||
|
||
|
||
no_sources = 0 | ||
multiple_sources = 0 | ||
ingested = 0 | ||
already_exists = 0 | ||
no_data = 0 | ||
|
||
# Ingest loop | ||
for source in uc_sheet_table: | ||
comment_filter = "" | ||
if len(source["designation_MKO"]) == 0: | ||
# instrument = "WFCAM" | ||
telescope = "UKIRT" | ||
filter_name = "WFCAM" | ||
comment_filter = "WFCAM filter is a guess. Check reference for actual filter and telescope used. " | ||
elif source["designation_MKO"][0] == "U": | ||
# instrument = "WFCAM" | ||
telescope = "UKIRT" | ||
filter_name = "WFCAM" | ||
elif source["designation_MKO"][0] == "V": | ||
# instrument = "VIRCAM" | ||
telescope = "VISTA" | ||
filter_name = "VISTA" | ||
else: | ||
# instrument = "WFCAM" | ||
telescope = "UKIRT" | ||
filter_name = "WFCAM" | ||
comment_filter = "WFCAM filter is a guess. Check reference for actual filter and telescope used. " | ||
match = None | ||
for band in ["Y", "J", "H", "K"]: | ||
measurement = source[f"{band}_MKO"] | ||
error = source[f"{band}err_MKO"] | ||
band_filter = f"{filter_name}.{band}" | ||
if band_filter == "VISTA.K": | ||
band_filter = "VISTA.Ks" | ||
|
||
if isnan(measurement): # skip if no data | ||
no_data += 1 | ||
continue | ||
uc_sheet_name = source["name"] | ||
if match is None: | ||
match = find_source_in_db( | ||
db, | ||
uc_sheet_name, | ||
ra=source["ra_j2000_formula"], | ||
dec=source["dec_j2000_formula"], | ||
) | ||
|
||
if len(match) == 1: | ||
# 1 Match found. INGEST! | ||
simple_source = match[0] | ||
logger.debug(f"Match found for {uc_sheet_name}: {simple_source}") | ||
|
||
try: | ||
references = source[f"ref_{band}_MKO"].split(";") | ||
reference = uc_ref_to_simple_ref(db, references[0]) | ||
|
||
comment_reference = "" | ||
if len(references) > 1: | ||
comment_reference = ( | ||
f"other references: {uc_ref_to_simple_ref(db, references[1])}" | ||
) | ||
if len(comment_filter) == 0 and len(comment_reference) == 0: | ||
comment = None | ||
else: | ||
comment = comment_filter + comment_reference | ||
ingest_Photometry( | ||
source=simple_source, | ||
band=band_filter, | ||
magnitude=measurement, | ||
mag_error=error, | ||
telescope=telescope, | ||
# instrument=instrument, | ||
comment=comment, | ||
reference=reference, | ||
raise_error=True, | ||
) | ||
ingested += 1 | ||
except AstroDBError as e: | ||
msg = "ingest failed with error: " + str(e) | ||
if "Photometry with same reference already exists" in str(e): | ||
already_exists += 1 | ||
else: | ||
logger.warning(msg) | ||
raise AstroDBError(msg) from e | ||
|
||
elif len(match) == 0: | ||
no_sources += 1 | ||
elif len(match) > 1: | ||
multiple_sources += 1 | ||
else: | ||
msg = "Unexpected situation occured" | ||
logger.error(msg) | ||
raise AstroDBError(msg) | ||
|
||
logger.info(f"ingested:{ingested}") # 7936 | ||
logger.info(f"already exists:{already_exists}") # 801 | ||
logger.info(f"no sources:{no_sources}") # 1403 | ||
logger.info(f"multiple sources:{multiple_sources}") # 8 | ||
logger.info(f"no data: {no_data}") # 5412 | ||
logger.info( | ||
f"data points tracked:{ingested+already_exists+no_sources+multiple_sources}" | ||
) | ||
total = ingested + already_exists + no_sources + multiple_sources + no_data | ||
logger.info(f"total: {total}") # 15560 | ||
|
||
if total != len(uc_sheet_table) * 4: | ||
msg = "data points tracked inconsistent with UC sheet" | ||
logger.error(msg) | ||
raise AstroDBError(msg) | ||
elif DB_SAVE: | ||
db.save_database(directory="data/") | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Very nice!