From 44e73ae1cbb50f51840fbc9600c3bfe02792cedb Mon Sep 17 00:00:00 2001 From: Brian Cherinka Date: Fri, 2 Aug 2024 16:02:10 -0400 Subject: [PATCH] Updates search endpoints to use the new observed column (#35) * included new observed column into model and pipe query * updating the main and cone search to use observed column * updating changelog * adding observed to carton/program searches * updating changelog --- CHANGELOG.rst | 2 ++ python/valis/db/models.py | 12 ++++++++-- python/valis/db/queries.py | 20 ++++++++++++---- python/valis/routes/query.py | 44 ++++++++++++++++++++---------------- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7e960fc..2102651 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Change Log * Adds endpoints for listing cartons and programs * Adds endpoints for target search by carton and program * Adds ``db_reset`` option to avoid closing and resetting the database connection +* Updates main, cone, and carton/program search to use new ``has_been_observed`` column; default is True. +* Updates the ``append_pipes`` query to return the new ``has_been_observed`` column 0.1.0 (10-24-2023) ------------------ diff --git a/python/valis/db/models.py b/python/valis/db/models.py index f56035f..d5ff842 100644 --- a/python/valis/db/models.py +++ b/python/valis/db/models.py @@ -7,7 +7,7 @@ import datetime import math from typing import Optional, Annotated, Any, TypeVar -from pydantic import ConfigDict, BaseModel, Field, BeforeValidator +from pydantic import ConfigDict, BaseModel, Field, BeforeValidator, field_validator, FieldValidationInfo from enum import Enum @@ -76,7 +76,15 @@ class SDSSidPipesBase(PeeweeBase): in_boss: bool = Field(..., description='Flag if target is in the BHM reductions', examples=[False]) in_apogee: bool = Field(..., description='Flag if target is in the MWM reductions', examples=[False]) in_astra: bool = Field(..., description='Flag if the target is in the Astra reductions', examples=[False]) - + has_been_observed: Optional[bool] = Field(False, validate_default=True, description='Flag if target has been observed or not', examples=[False]) + + @field_validator('has_been_observed') + @classmethod + def is_observed(cls, v: str, info: FieldValidationInfo) -> str: + """ validator for when has_been_observed was not available in table """ + if not v: + return info.data['in_boss'] or info.data['in_apogee'] or info.data['in_astra'] + return v class SDSSModel(SDSSidStackedBase, SDSSidPipesBase): """ Main Pydantic response for SDSS id plus Pipes flags """ diff --git a/python/valis/db/queries.py b/python/valis/db/queries.py index 6ea11ce..ac0c526 100644 --- a/python/valis/db/queries.py +++ b/python/valis/db/queries.py @@ -25,7 +25,8 @@ from valis.utils.versions import get_software_tag -def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.ModelSelect: +def append_pipes(query: peewee.ModelSelect, table: str = 'stacked', + observed: bool = True) -> peewee.ModelSelect: """ Joins a query to the SDSSidToPipes table Joines an existing query to the SDSSidToPipes table and returns @@ -40,6 +41,8 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.Mo the input query to join to table : str, optional the type of sdss_id table joining to, by default 'stacked' + observed : bool, optional + Flag to filter on observed targets, by default True Returns ------- @@ -55,12 +58,18 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.Mo raise ValueError('table must be either "stacked" or "flat"') model = vizdb.SDSSidStacked if table == 'stacked' else vizdb.SDSSidFlat - return query.select_extend(vizdb.SDSSidToPipes.in_boss, + qq = query.select_extend(vizdb.SDSSidToPipes.in_boss, vizdb.SDSSidToPipes.in_apogee, - vizdb.SDSSidToPipes.in_astra).\ + vizdb.SDSSidToPipes.in_astra, + vizdb.SDSSidToPipes.has_been_observed).\ join(vizdb.SDSSidToPipes, on=(model.sdss_id == vizdb.SDSSidToPipes.sdss_id), attr='pipes').distinct(vizdb.SDSSidToPipes.sdss_id) + if observed: + qq = qq.where(vizdb.SDSSidToPipes.has_been_observed == observed) + + return qq + def get_pipes(sdss_id: int) -> peewee.ModelSelect: """ Get the pipelines for an sdss_id @@ -173,11 +182,12 @@ def get_targets_by_sdss_id(sdss_id: Union[int, list[int]] = []) -> peewee.ModelS peewee.ModelSelect the ORM query """ - if type(sdss_id) == int: + if type(sdss_id) is int: sdss_id = [sdss_id] return vizdb.SDSSidStacked.select().where(vizdb.SDSSidStacked.sdss_id.in_(sdss_id)) + def get_targets_by_catalog_id(catalog_id: int) -> peewee.ModelSelect: """ Perform a search for SDSS targets on vizdb.SDSSidStacked based on the catalog_id. @@ -265,7 +275,7 @@ def carton_program_search(name: str, """ if query is None: - query = vizdb.SDSSidStacked.select(peewee.fn.DISTINCT(vizdb.SDSSidStacked.sdss_id)) + query = vizdb.SDSSidStacked.select(vizdb.SDSSidStacked).distinct() query = (query.join( vizdb.SDSSidFlat, diff --git a/python/valis/routes/query.py b/python/valis/routes/query.py index 31e42df..e034af4 100644 --- a/python/valis/routes/query.py +++ b/python/valis/routes/query.py @@ -10,7 +10,7 @@ from valis.routes.base import Base from valis.db.db import get_pw_db -from valis.db.models import SDSSidStackedBase, SDSSidPipesBase, MapperName +from valis.db.models import SDSSidStackedBase, SDSSidPipesBase, MapperName, SDSSModel from valis.db.queries import (cone_search, append_pipes, carton_program_search, carton_program_list, carton_program_map, get_targets_by_sdss_id, get_targets_by_catalog_id, @@ -30,16 +30,16 @@ class SearchCoordUnits(str, Enum): class SearchModel(BaseModel): """ Input main query body model """ - ra: Optional[Union[float, str]] = Field(None, description='Right Ascension in degrees or hmsdms', example=315.01417) - dec: Optional[Union[float, str]] = Field(None, description='Declination in degrees or hmsdms', example=35.299) - radius: Optional[Float] = Field(None, description='Search radius in specified units', example=0.01) + ra: Optional[Union[float, str]] = Field(None, description='Right Ascension in degrees or hmsdms', example=150.385) + dec: Optional[Union[float, str]] = Field(None, description='Declination in degrees or hmsdms', example=1.02) + radius: Optional[Float] = Field(None, description='Search radius in specified units', example=0.02) units: Optional[SearchCoordUnits] = Field('degree', description='Units of search radius', example='degree') id: Optional[Union[int, str]] = Field(None, description='The SDSS identifier', example=23326) program: Optional[str] = Field(None, description='The program name', example='bhm_rm') carton: Optional[str] = Field(None, description='The carton name', example='bhm_rm_core') + observed: Optional[bool] = Field(True, description='Flag to only include targets that have been observed', example=True) - -class MainResponse(SDSSidPipesBase, SDSSidStackedBase): +class MainResponse(SDSSModel): """ Combined model from all individual query models """ @@ -97,20 +97,23 @@ async def main_search(self, body: SearchModel): 'program' if body.program else 'carton', query=query) # append query to pipes - query = append_pipes(query) + query = append_pipes(query, observed=body.observed) return {'status': 'success', 'data': query.dicts().iterator(), 'msg': 'data successfully retrieved'} @router.get('/cone', summary='Perform a cone search for SDSS targets with sdss_ids', - response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) + response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)]) async def cone_search(self, - ra: Annotated[Union[float, str], Query(description='Right Ascension in degrees or hmsdms', example=315.01417)], - dec: Annotated[Union[float, str], Query(description='Declination in degrees or hmsdms', example=35.299)], - radius: Annotated[float, Query(description='Search radius in specified units', example=0.01)], - units: Annotated[SearchCoordUnits, Query(description='Units of search radius', example='degree')] = "degree"): + ra: Annotated[Union[float, str], Query(description='Right Ascension in degrees or hmsdms', example=315.78)], + dec: Annotated[Union[float, str], Query(description='Declination in degrees or hmsdms', example=-3.2)], + radius: Annotated[float, Query(description='Search radius in specified units', example=0.02)], + units: Annotated[SearchCoordUnits, Query(description='Units of search radius', example='degree')] = "degree", + observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True): """ Perform a cone search """ - return list(cone_search(ra, dec, radius, units=units)) + res = cone_search(ra, dec, radius, units=units) + r = append_pipes(res, observed=observed) + return r.dicts().iterator() @router.get('/sdssid', summary='Perform a search for an SDSS target based on the sdss_id', response_model=Union[SDSSidStackedBase, dict], dependencies=[Depends(get_pw_db)]) @@ -135,7 +138,7 @@ async def sdss_id_search(self, sdss_id: Annotated[int, Query(description='Value async def sdss_ids_search(self, body: SDSSIdsModel): """ Perform a search for SDSS targets based on a list of input sdss_id values.""" return list(get_targets_by_sdss_id(body.sdss_id_list)) - + @router.get('/catalogid', summary='Perform a search for SDSS targets based on the catalog_id', response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) async def catalog_id_search(self, catalog_id: Annotated[int, Query(description='Value of catalog_id', example=7613823349)]): @@ -165,17 +168,20 @@ async def program_map(self): return carton_program_map() @router.get('/carton-program', summary='Search for all SDSS targets within a carton or program', - response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) + response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)]) async def carton_program(self, name: Annotated[str, Query(description='Carton or program name', example='manual_mwm_tess_ob')], name_type: Annotated[str, Query(enum=['carton', 'program'], description='Specify search on carton or program', - example='carton')] = 'carton'): + example='carton')] = 'carton', + observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True): """ Perform a search on carton or program """ - with database.atomic() as transaction: + with database.atomic(): database.execute_sql('SET LOCAL enable_seqscan=false;') - return list(carton_program_search(name, name_type)) + query = carton_program_search(name, name_type) + query = append_pipes(query, observed=observed) + return query.dicts().iterator() @router.get('/obs', summary='Return targets with spectrum at observatory', response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) @@ -195,7 +201,7 @@ async def obs(self, @router.get('/mapper', summary='Perform a search for SDSS targets based on the mapper', response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)]) - async def get_target_list_by_mapper(self, + async def get_target_list_by_mapper(self, mapper: Annotated[MapperName, Query(description='Mapper name', example=MapperName.MWM)] = MapperName.MWM, page_number: Annotated[int, Query(description='Page number of the returned items', gt=0, example=1)] = 1, items_per_page: Annotated[int, Query(description='Number of items displayed in a page', gt=0, example=10)] = 10):