diff --git a/app.py b/app.py index c4fa29e7..d62a6c44 100644 --- a/app.py +++ b/app.py @@ -8,8 +8,11 @@ from resources.RequestResetPassword import RequestResetPassword from resources.ResetPassword import ResetPassword from resources.QuickSearch import QuickSearch -from resources.DataSources import DataSources -from resources.DataSources import DataSourceById +from resources.DataSources import ( + DataSources, + DataSourcesNeedsIdentification, + DataSourceById, +) from resources.Agencies import Agencies from resources.Archives import Archives from resources.SearchTokens import SearchTokens @@ -62,6 +65,11 @@ "/data-sources", resource_class_kwargs={"psycopg2_connection": psycopg2_connection}, ) +api.add_resource( + DataSourcesNeedsIdentification, + "/data-sources-needs-identification", + resource_class_kwargs={"psycopg2_connection": psycopg2_connection}, +) api.add_resource( DataSourceById, "/data-sources-by-id/", diff --git a/app_test.py b/app_test.py index 589693d3..c083469f 100644 --- a/app_test.py +++ b/app_test.py @@ -9,7 +9,8 @@ ) from middleware.data_source_queries import ( data_sources_query, - data_sources_results, + approved_data_sources, + needs_identification_data_sources, data_source_by_id_query, data_source_by_id_results, DATA_SOURCES_APPROVED_COLUMNS, @@ -109,13 +110,19 @@ def test_unaltered_search_query(session): def test_data_sources(session): - response = data_sources_results(conn=session) + response = approved_data_sources(conn=session) + + assert response + + +def test_needs_identification(session): + response = needs_identification_data_sources(conn=session) assert response def test_data_sources_approved(session): - response = data_sources_results(conn=session) + response = approved_data_sources(conn=session) assert ( len([d for d in response if "https://joinstatepolice.ny.gov/15-mile-run" in d]) diff --git a/app_test_data.py b/app_test_data.py index 12c4b414..4cd664c5 100644 --- a/app_test_data.py +++ b/app_test_data.py @@ -103,6 +103,57 @@ "ok", "approved", ), + ( + "Media Bulletins for ", + "NULL", + "NULL", + "Media Bulletins", + "https://hollister.ca.gov/government/city-departments/police/", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + datetime.datetime(2023, 10, 26, 18, 20, 54, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 5, 30, tzinfo=datetime.timezone.utc), + "NULL", + "NULL", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "needs identification", + ), ] DATA_SOURCE_QUERY_RESULTS = [ ( diff --git a/middleware/data_source_queries.py b/middleware/data_source_queries.py index 3d950f73..4c4624cf 100644 --- a/middleware/data_source_queries.py +++ b/middleware/data_source_queries.py @@ -194,20 +194,54 @@ def data_sources_results(conn: sqlite3.Connection) -> List[Dict[str, Any]]: return results - -def data_sources_query(conn: Optional[sqlite3.Connection] = None, - test_query_results: Optional[List[Dict[str, Any]]] = None) -> List[Dict[str, Any]]: + def needs_identification_data_sources(conn) -> list: + """ + Returns a list of data sources that need identification """ + cursor = conn.cursor() + joined_column_names = ", ".join(DATA_SOURCES_APPROVED_COLUMNS) + + sql_query = """ + SELECT + {} + FROM + data_sources + WHERE + approval_status = 'needs identification' + """.format( + joined_column_names + ) + cursor.execute(sql_query) + results = cursor.fetchall() + cursor.close() + + return results + + + + +def data_sources_query( + conn: Optional[sqlite3.Connection] = None + test_query_results: Optional[List[Dict[str, Any]]] = [], + approval_status="approved" +) -> List[Dict[str, Any]]: + """ Queries and processes a list of approved data sources, optionally using test data. Parameters: - conn: Optional; sqlite3.Connection object for database access if test_query_results is not provided. - test_query_results: Optional; predefined results for testing purposes. + - approval_status: Whether to get approved data sources or not. Defaults to "approved" Returns: - A list of dictionaries, each containing details of an approved data source and its associated agency name. """ - results = data_sources_results(conn, "", "") if conn else test_query_results + if conn and approval_status == "approved": + results = approved_data_sources(conn) + elif conn: + results = needs_identification_data_sources(conn) + else: + results = test_query_results data_source_output_columns = DATA_SOURCES_APPROVED_COLUMNS + ["agency_name"] diff --git a/resources/DataSources.py b/resources/DataSources.py index 70c5f80a..32e10267 100644 --- a/resources/DataSources.py +++ b/resources/DataSources.py @@ -4,7 +4,7 @@ from middleware.data_source_queries import data_source_by_id_query, data_sources_query import json from datetime import datetime -from utilities.common import convert_dates_to_strings + import uuid @@ -79,7 +79,9 @@ def __init__(self, **kwargs): @api_required def get(self): try: - data_source_matches = data_sources_query(self.psycopg2_connection) + data_source_matches = data_sources_query( + self.psycopg2_connection, [], "approved" + ) data_sources = { "count": len(data_source_matches), @@ -136,3 +138,27 @@ def post(self): self.psycopg2_connection.rollback() print(str(e)) return {"message": "There has been an error adding the data source"}, 500 + + +class DataSourcesNeedsIdentification(Resource): + def __init__(self, **kwargs): + self.psycopg2_connection = kwargs["psycopg2_connection"] + + @api_required + def get(self): + try: + data_source_matches = data_sources_query( + self.psycopg2_connection, [], "needs_identification" + ) + + data_sources = { + "count": len(data_source_matches), + "data": data_source_matches, + } + + return data_sources + + except Exception as e: + self.psycopg2_connection.rollback() + print(str(e)) + return {"message": "There has been an error pulling data!"}, 500