Updates search endpoints to use the new observed column (#35)

* included new observed column into model and pipe query

* updating the main and cone search to use observed column

* updating changelog

* adding observed to carton/program searches

* updating changelog

CHANGELOG.rst

@@ -12,6 +12,8 @@ Change Log
 * Adds endpoints for listing cartons and programs
 * Adds endpoints for target search by carton and program
 * Adds ``db_reset`` option to avoid closing and resetting the database connection
+* Updates main, cone, and carton/program search to use new ``has_been_observed`` column; default is True.
+* Updates the ``append_pipes`` query to return the new ``has_been_observed`` column
 
 0.1.0 (10-24-2023)
 ------------------

python/valis/db/models.py

@@ -7,7 +7,7 @@
 import datetime
 import math
 from typing import Optional, Annotated, Any, TypeVar
-from pydantic import ConfigDict, BaseModel, Field, BeforeValidator
+from pydantic import ConfigDict, BaseModel, Field, BeforeValidator, field_validator, FieldValidationInfo
 from enum import Enum
 
 
@@ -76,7 +76,15 @@ class SDSSidPipesBase(PeeweeBase):
     in_boss: bool = Field(..., description='Flag if target is in the BHM reductions', examples=[False])
     in_apogee: bool = Field(..., description='Flag if target is in the MWM reductions', examples=[False])
     in_astra: bool = Field(..., description='Flag if the target is in the Astra reductions', examples=[False])
-
+    has_been_observed: Optional[bool] = Field(False, validate_default=True, description='Flag if target has been observed or not', examples=[False])
+
+    @field_validator('has_been_observed')
+    @classmethod
+    def is_observed(cls, v: str, info: FieldValidationInfo) -> str:
+        """ validator for when has_been_observed was not available in table """
+        if not v:
+            return info.data['in_boss'] or info.data['in_apogee'] or info.data['in_astra']
+        return v
 
 class SDSSModel(SDSSidStackedBase, SDSSidPipesBase):
     """ Main Pydantic response for SDSS id plus Pipes flags """

python/valis/db/queries.py

@@ -25,7 +25,8 @@
 from valis.utils.versions import get_software_tag
 
 
-def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.ModelSelect:
+def append_pipes(query: peewee.ModelSelect, table: str = 'stacked',
+                 observed: bool = True) -> peewee.ModelSelect:
     """ Joins a query to the SDSSidToPipes table
 
     Joines an existing query to the SDSSidToPipes table and returns
@@ -40,6 +41,8 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.Mo
         the input query to join to
     table : str, optional
         the type of sdss_id table joining to, by default 'stacked'
+    observed : bool, optional
+        Flag to filter on observed targets, by default True
 
     Returns
     -------
@@ -55,12 +58,18 @@ def append_pipes(query: peewee.ModelSelect, table: str = 'stacked') -> peewee.Mo
         raise ValueError('table must be either "stacked" or "flat"')
 
     model = vizdb.SDSSidStacked if table == 'stacked' else vizdb.SDSSidFlat
-    return query.select_extend(vizdb.SDSSidToPipes.in_boss,
+    qq = query.select_extend(vizdb.SDSSidToPipes.in_boss,
                                vizdb.SDSSidToPipes.in_apogee,
-                               vizdb.SDSSidToPipes.in_astra).\
+                               vizdb.SDSSidToPipes.in_astra,
+                               vizdb.SDSSidToPipes.has_been_observed).\
         join(vizdb.SDSSidToPipes, on=(model.sdss_id == vizdb.SDSSidToPipes.sdss_id),
              attr='pipes').distinct(vizdb.SDSSidToPipes.sdss_id)
 
+    if observed:
+        qq = qq.where(vizdb.SDSSidToPipes.has_been_observed == observed)
+
+    return qq
+
 
 def get_pipes(sdss_id: int) -> peewee.ModelSelect:
     """ Get the pipelines for an sdss_id
@@ -173,11 +182,12 @@ def get_targets_by_sdss_id(sdss_id: Union[int, list[int]] = []) -> peewee.ModelS
     peewee.ModelSelect
         the ORM query
     """
-    if type(sdss_id) == int:
+    if type(sdss_id) is int:
         sdss_id = [sdss_id]
 
     return vizdb.SDSSidStacked.select().where(vizdb.SDSSidStacked.sdss_id.in_(sdss_id))
 
+
 def get_targets_by_catalog_id(catalog_id: int) -> peewee.ModelSelect:
     """ Perform a search for SDSS targets on vizdb.SDSSidStacked based on the catalog_id.
 
@@ -265,7 +275,7 @@ def carton_program_search(name: str,
     """
 
     if query is None:
-        query = vizdb.SDSSidStacked.select(peewee.fn.DISTINCT(vizdb.SDSSidStacked.sdss_id))
+        query = vizdb.SDSSidStacked.select(vizdb.SDSSidStacked).distinct()
 
     query = (query.join(
                 vizdb.SDSSidFlat,

python/valis/routes/query.py

@@ -10,7 +10,7 @@
 
 from valis.routes.base import Base
 from valis.db.db import get_pw_db
-from valis.db.models import SDSSidStackedBase, SDSSidPipesBase, MapperName
+from valis.db.models import SDSSidStackedBase, SDSSidPipesBase, MapperName, SDSSModel
 from valis.db.queries import (cone_search, append_pipes, carton_program_search,
                               carton_program_list, carton_program_map,
                               get_targets_by_sdss_id, get_targets_by_catalog_id,
@@ -30,16 +30,16 @@ class SearchCoordUnits(str, Enum):
 
 class SearchModel(BaseModel):
     """ Input main query body model """
-    ra: Optional[Union[float, str]] = Field(None, description='Right Ascension in degrees or hmsdms', example=315.01417)
-    dec: Optional[Union[float, str]] = Field(None, description='Declination in degrees or hmsdms', example=35.299)
-    radius: Optional[Float] = Field(None, description='Search radius in specified units', example=0.01)
+    ra: Optional[Union[float, str]] = Field(None, description='Right Ascension in degrees or hmsdms', example=150.385)
+    dec: Optional[Union[float, str]] = Field(None, description='Declination in degrees or hmsdms', example=1.02)
+    radius: Optional[Float] = Field(None, description='Search radius in specified units', example=0.02)
     units: Optional[SearchCoordUnits] = Field('degree', description='Units of search radius', example='degree')
     id: Optional[Union[int, str]] = Field(None, description='The SDSS identifier', example=23326)
     program: Optional[str] = Field(None, description='The program name', example='bhm_rm')
     carton: Optional[str] = Field(None, description='The carton name', example='bhm_rm_core')
+    observed: Optional[bool] = Field(True, description='Flag to only include targets that have been observed', example=True)
 
-
-class MainResponse(SDSSidPipesBase, SDSSidStackedBase):
+class MainResponse(SDSSModel):
     """ Combined model from all individual query models """
 
 
@@ -97,20 +97,23 @@ async def main_search(self, body: SearchModel):
                                           'program' if body.program else 'carton',
                                           query=query)
         # append query to pipes
-        query = append_pipes(query)
+        query = append_pipes(query, observed=body.observed)
 
         return {'status': 'success', 'data': query.dicts().iterator(), 'msg': 'data successfully retrieved'}
 
     @router.get('/cone', summary='Perform a cone search for SDSS targets with sdss_ids',
-                response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)])
+                response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)])
     async def cone_search(self,
-                          ra: Annotated[Union[float, str], Query(description='Right Ascension in degrees or hmsdms', example=315.01417)],
-                          dec: Annotated[Union[float, str], Query(description='Declination in degrees or hmsdms', example=35.299)],
-                          radius: Annotated[float, Query(description='Search radius in specified units', example=0.01)],
-                          units: Annotated[SearchCoordUnits, Query(description='Units of search radius', example='degree')] = "degree"):
+                          ra: Annotated[Union[float, str], Query(description='Right Ascension in degrees or hmsdms', example=315.78)],
+                          dec: Annotated[Union[float, str], Query(description='Declination in degrees or hmsdms', example=-3.2)],
+                          radius: Annotated[float, Query(description='Search radius in specified units', example=0.02)],
+                          units: Annotated[SearchCoordUnits, Query(description='Units of search radius', example='degree')] = "degree",
+                          observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True):
         """ Perform a cone search """
 
-        return list(cone_search(ra, dec, radius, units=units))
+        res = cone_search(ra, dec, radius, units=units)
+        r = append_pipes(res, observed=observed)
+        return r.dicts().iterator()
 
     @router.get('/sdssid', summary='Perform a search for an SDSS target based on the sdss_id',
                 response_model=Union[SDSSidStackedBase, dict], dependencies=[Depends(get_pw_db)])
@@ -135,7 +138,7 @@ async def sdss_id_search(self, sdss_id: Annotated[int, Query(description='Value
     async def sdss_ids_search(self, body: SDSSIdsModel):
         """ Perform a search for SDSS targets based on a list of input sdss_id values."""
         return list(get_targets_by_sdss_id(body.sdss_id_list))
-    
+
     @router.get('/catalogid', summary='Perform a search for SDSS targets based on the catalog_id',
                 response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)])
     async def catalog_id_search(self, catalog_id: Annotated[int, Query(description='Value of catalog_id', example=7613823349)]):
@@ -165,17 +168,20 @@ async def program_map(self):
         return carton_program_map()
 
     @router.get('/carton-program', summary='Search for all SDSS targets within a carton or program',
-                response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)])
+                response_model=List[SDSSModel], dependencies=[Depends(get_pw_db)])
     async def carton_program(self,
                              name: Annotated[str, Query(description='Carton or program name', example='manual_mwm_tess_ob')],
                              name_type: Annotated[str,
                                                   Query(enum=['carton', 'program'],
                                                         description='Specify search on carton or program',
-                                                        example='carton')] = 'carton'):
+                                                        example='carton')] = 'carton',
+                             observed: Annotated[bool, Query(description='Flag to only include targets that have been observed', example=True)] = True):
         """ Perform a search on carton or program """
-        with database.atomic() as transaction:
+        with database.atomic():
             database.execute_sql('SET LOCAL enable_seqscan=false;')
-            return list(carton_program_search(name, name_type))
+            query = carton_program_search(name, name_type)
+            query = append_pipes(query, observed=observed)
+            return query.dicts().iterator()
 
     @router.get('/obs', summary='Return targets with spectrum at observatory',
                 response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)])
@@ -195,7 +201,7 @@ async def obs(self,
 
     @router.get('/mapper', summary='Perform a search for SDSS targets based on the mapper',
                 response_model=List[SDSSidStackedBase], dependencies=[Depends(get_pw_db)])
-    async def get_target_list_by_mapper(self, 
+    async def get_target_list_by_mapper(self,
                                         mapper: Annotated[MapperName, Query(description='Mapper name', example=MapperName.MWM)] = MapperName.MWM,
                                         page_number: Annotated[int, Query(description='Page number of the returned items', gt=0, example=1)] = 1,
                                         items_per_page: Annotated[int, Query(description='Number of items displayed in a page', gt=0, example=10)] = 10):