From 1bbcb559be77a2645e20a566cde7c6b37a7fb7ca Mon Sep 17 00:00:00 2001 From: Eric Rosas Date: Tue, 1 Oct 2024 15:54:45 -0700 Subject: [PATCH 1/3] Added `metadata:` columns to manifest functions Added `metadata:` columns to manifest functions --- utils.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/utils.py b/utils.py index 866ab83..216858d 100644 --- a/utils.py +++ b/utils.py @@ -295,6 +295,9 @@ def make_manifest_with_calexp_images( cutouts = [] # for each moment in time, create the calexp image for i, idx in enumerate(idx_select): + if hasattr(sorted_sources, "diaObjectId") is False: + print("The column 'diaObjectId' is required to send data to the Zooniverse for this notebook! Please query for your data again adding 'diaObjectId' and then rerun this cell.") + return star_ra = sorted_sources["ra"][idx] star_dec = sorted_sources["decl"][idx] star_visitid = sorted_sources["visitId"][idx] @@ -316,6 +319,8 @@ def make_manifest_with_calexp_images( ) # and of the diaObjectId figout_data["diaObjectId:image_" + str(i)] = str(star_id) + figout_data[f"metadata:diaObjectId_image_{str(i)}"] = str(star_id) + figout_data["filename"] = str(star_id) + "_" + str(star_ccdid) + ".png" cutouts.append(figout_data) return cutouts @@ -324,6 +329,7 @@ def make_manifest_with_calexp_images( def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): # In-memory manifest file as an array of dicts manifest = [] + has_canonical_id = False # Create directory if it does not already exist if os.path.isdir(batch_dir) is False: @@ -340,10 +346,6 @@ def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): csv_row = { # required column, do not change the column name "filename": filename, - # required column, do not change the column name - "objectId": row.objectId, - # required column, do not change the column name - "objectIdType": "DIRECT", # Add your desired columns: "coord_ra": row.coord_ra, "coord_dec": row.coord_dec, @@ -352,8 +354,25 @@ def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): "r_extendedness": row.r_extendedness, "r_inputCount": row.r_inputCount, } + + # These columns are required in order to cross-match your completed + if hasattr(row, "objectId"): + has_canonical_id = True + csv_row["objectId"] = row.objectId + csv_row["metadata:objectId"] = row.objectId + csv_row["objectIdType"] = "DIRECT" + if hasattr(row, "diaObjectId"): + has_canonical_id = True + csv_row["diaObjectId"] = row.diaObjectId + csv_row["metadata:diaObjectId"] = row.diaObjectId + if "objectIdType" not in csv_row: + csv_row["objectIdType"] = "INDIRECT" + manifest.append(csv_row) remove_figure(figout) + + if has_canonical_id is False: + print("WARNING! You did not include either objectId or diaObjectId in your manifest file dataset. These fields are used to cross-match the completed classifications workflow data back to the original data. Consider rerunning this cell after adding either objectId, diaObjectId, or both.") return manifest From 16b2a4a23fe71f5f81b927d48ae65c5de888ad63 Mon Sep 17 00:00:00 2001 From: Eric Rosas Date: Tue, 1 Oct 2024 16:07:50 -0700 Subject: [PATCH 2/3] Made Flake8 happy Made Flake8 happy --- utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/utils.py b/utils.py index 216858d..cf4a59e 100644 --- a/utils.py +++ b/utils.py @@ -296,7 +296,9 @@ def make_manifest_with_calexp_images( # for each moment in time, create the calexp image for i, idx in enumerate(idx_select): if hasattr(sorted_sources, "diaObjectId") is False: - print("The column 'diaObjectId' is required to send data to the Zooniverse for this notebook! Please query for your data again adding 'diaObjectId' and then rerun this cell.") + print("The column 'diaObjectId' is required to send data to the Zooniverse" + + "for this notebook! Please query for your data again adding " + + "'diaObjectId' and then rerun this cell.") return star_ra = sorted_sources["ra"][idx] star_dec = sorted_sources["decl"][idx] @@ -320,7 +322,7 @@ def make_manifest_with_calexp_images( # and of the diaObjectId figout_data["diaObjectId:image_" + str(i)] = str(star_id) figout_data[f"metadata:diaObjectId_image_{str(i)}"] = str(star_id) - + figout_data["filename"] = str(star_id) + "_" + str(star_ccdid) + ".png" cutouts.append(figout_data) return cutouts @@ -355,7 +357,7 @@ def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): "r_inputCount": row.r_inputCount, } - # These columns are required in order to cross-match your completed + # These columns are required in order to cross-match your completed if hasattr(row, "objectId"): has_canonical_id = True csv_row["objectId"] = row.objectId @@ -367,12 +369,16 @@ def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): csv_row["metadata:diaObjectId"] = row.diaObjectId if "objectIdType" not in csv_row: csv_row["objectIdType"] = "INDIRECT" - + manifest.append(csv_row) remove_figure(figout) if has_canonical_id is False: - print("WARNING! You did not include either objectId or diaObjectId in your manifest file dataset. These fields are used to cross-match the completed classifications workflow data back to the original data. Consider rerunning this cell after adding either objectId, diaObjectId, or both.") + print("WARNING! You did not include either objectId or diaObjectId in your " + + "manifest file dataset. These fields are used to cross-match the " + + "completed classifications workflow data back to the original data. " + + "Consider rerunning this cell after adding either objectId, " + + "diaObjectId, or both.") return manifest From 01a0f2454297c3456dc69c58a35e60378a3b7385 Mon Sep 17 00:00:00 2001 From: Eric Rosas Date: Wed, 2 Oct 2024 15:41:36 -0700 Subject: [PATCH 3/3] Making Flake8 even happier Making Flake8 even happier --- utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils.py b/utils.py index cf4a59e..d48ee32 100644 --- a/utils.py +++ b/utils.py @@ -296,9 +296,9 @@ def make_manifest_with_calexp_images( # for each moment in time, create the calexp image for i, idx in enumerate(idx_select): if hasattr(sorted_sources, "diaObjectId") is False: - print("The column 'diaObjectId' is required to send data to the Zooniverse" + - "for this notebook! Please query for your data again adding " + - "'diaObjectId' and then rerun this cell.") + print("The column 'diaObjectId' is required to send data to the Zooniverse" + + "for this notebook! Please query for your data again adding " + + "'diaObjectId' and then rerun this cell.") return star_ra = sorted_sources["ra"][idx] star_dec = sorted_sources["decl"][idx] @@ -374,11 +374,11 @@ def make_manifest_with_deepcoadd_images(results_table, butler, batch_dir): remove_figure(figout) if has_canonical_id is False: - print("WARNING! You did not include either objectId or diaObjectId in your " + - "manifest file dataset. These fields are used to cross-match the " + - "completed classifications workflow data back to the original data. " + - "Consider rerunning this cell after adding either objectId, " + - "diaObjectId, or both.") + print("WARNING! You did not include either objectId or diaObjectId in your " + + "manifest file dataset. These fields are used to cross-match the " + + "completed classifications workflow data back to the original data. " + + "Consider rerunning this cell after adding either objectId, " + + "diaObjectId, or both.") return manifest