Merge branch 'dev' into mc_type_hinting_and_docstring_middleware

app.py

@@ -8,8 +8,11 @@
 from resources.RequestResetPassword import RequestResetPassword
 from resources.ResetPassword import ResetPassword
 from resources.QuickSearch import QuickSearch
-from resources.DataSources import DataSources
-from resources.DataSources import DataSourceById
+from resources.DataSources import (
+    DataSources,
+    DataSourcesNeedsIdentification,
+    DataSourceById,
+)
 from resources.Agencies import Agencies
 from resources.Archives import Archives
 from resources.SearchTokens import SearchTokens
@@ -62,6 +65,11 @@
     "/data-sources",
     resource_class_kwargs={"psycopg2_connection": psycopg2_connection},
 )
+api.add_resource(
+    DataSourcesNeedsIdentification,
+    "/data-sources-needs-identification",
+    resource_class_kwargs={"psycopg2_connection": psycopg2_connection},
+)
 api.add_resource(
     DataSourceById,
     "/data-sources-by-id/<data_source_id>",

app_test.py

@@ -9,7 +9,8 @@
 )
 from middleware.data_source_queries import (
     data_sources_query,
-    data_sources_results,
+    approved_data_sources,
+    needs_identification_data_sources,
     data_source_by_id_query,
     data_source_by_id_results,
     DATA_SOURCES_APPROVED_COLUMNS,
@@ -109,13 +110,19 @@ def test_unaltered_search_query(session):
 
 
 def test_data_sources(session):
-    response = data_sources_results(conn=session)
+    response = approved_data_sources(conn=session)
+
+    assert response
+
+
+def test_needs_identification(session):
+    response = needs_identification_data_sources(conn=session)
 
     assert response
 
 
 def test_data_sources_approved(session):
-    response = data_sources_results(conn=session)
+    response = approved_data_sources(conn=session)
 
     assert (
         len([d for d in response if "https://joinstatepolice.ny.gov/15-mile-run" in d])

app_test_data.py

@@ -103,6 +103,57 @@
         "ok",
         "approved",
     ),
+    (
+        "Media Bulletins for ",
+        "NULL",
+        "NULL",
+        "Media Bulletins",
+        "https://hollister.ca.gov/government/city-departments/police/",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        datetime.datetime(2023, 10, 26, 18, 20, 54, tzinfo=datetime.timezone.utc),
+        datetime.datetime(2023, 11, 8, 19, 5, 30, tzinfo=datetime.timezone.utc),
+        "NULL",
+        "NULL",
+        "NULL",
+        '{"id": "usrtLIB4Vr3jTH8Ro", "email": "[email protected]", "name": "Josh Chamberlain"}',
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "NULL",
+        "needs identification",
+    ),
 ]
 DATA_SOURCE_QUERY_RESULTS = [
     (

middleware/data_source_queries.py

@@ -194,20 +194,54 @@ def data_sources_results(conn: sqlite3.Connection) -> List[Dict[str, Any]]:
 
     return results
 
-
-def data_sources_query(conn: Optional[sqlite3.Connection] = None,
-                       test_query_results: Optional[List[Dict[str, Any]]] = None) -> List[Dict[str, Any]]:
+  def needs_identification_data_sources(conn) -> list:
+    """
+    Returns a list of data sources that need identification
     """
+    cursor = conn.cursor()
+    joined_column_names = ", ".join(DATA_SOURCES_APPROVED_COLUMNS)
+
+    sql_query = """
+        SELECT
+            {}
+        FROM
+            data_sources
+        WHERE
+            approval_status = 'needs identification'
+    """.format(
+        joined_column_names
+    )
+    cursor.execute(sql_query)
+    results = cursor.fetchall()
+    cursor.close()
+
+    return results
+
+
+
+
+def data_sources_query(
+  conn: Optional[sqlite3.Connection] = None
+  test_query_results: Optional[List[Dict[str, Any]]] = [], 
+  approval_status="approved"
+) -> List[Dict[str, Any]]:
+ """
     Queries and processes a list of approved data sources, optionally using test data.
 
     Parameters:
     - conn: Optional; sqlite3.Connection object for database access if test_query_results is not provided.
     - test_query_results: Optional; predefined results for testing purposes.
+    - approval_status: Whether to get approved data sources or not. Defaults to "approved"
 
     Returns:
     - A list of dictionaries, each containing details of an approved data source and its associated agency name.
     """
-    results = data_sources_results(conn, "", "") if conn else test_query_results
+    if conn and approval_status == "approved":
+        results = approved_data_sources(conn)
+    elif conn:
+        results = needs_identification_data_sources(conn)
+    else:
+        results = test_query_results
 
     data_source_output_columns = DATA_SOURCES_APPROVED_COLUMNS + ["agency_name"]
 

resources/DataSources.py

@@ -4,7 +4,7 @@
 from middleware.data_source_queries import data_source_by_id_query, data_sources_query
 import json
 from datetime import datetime
-from utilities.common import convert_dates_to_strings
+
 import uuid
 
 
@@ -79,7 +79,9 @@ def __init__(self, **kwargs):
     @api_required
     def get(self):
         try:
-            data_source_matches = data_sources_query(self.psycopg2_connection)
+            data_source_matches = data_sources_query(
+                self.psycopg2_connection, [], "approved"
+            )
 
             data_sources = {
                 "count": len(data_source_matches),
@@ -136,3 +138,27 @@ def post(self):
             self.psycopg2_connection.rollback()
             print(str(e))
             return {"message": "There has been an error adding the data source"}, 500
+
+
+class DataSourcesNeedsIdentification(Resource):
+    def __init__(self, **kwargs):
+        self.psycopg2_connection = kwargs["psycopg2_connection"]
+
+    @api_required
+    def get(self):
+        try:
+            data_source_matches = data_sources_query(
+                self.psycopg2_connection, [], "needs_identification"
+            )
+
+            data_sources = {
+                "count": len(data_source_matches),
+                "data": data_source_matches,
+            }
+
+            return data_sources
+
+        except Exception as e:
+            self.psycopg2_connection.rollback()
+            print(str(e))
+            return {"message": "There has been an error pulling data!"}, 500