From d3b92f1ccdcc2bd6f39f1597a5a48de22dfc11ec Mon Sep 17 00:00:00 2001 From: Marty Bode Date: Fri, 12 Jan 2024 08:49:07 -0500 Subject: [PATCH 1/5] wip --- app_test.py | 96 ++++++++++++++++--------------- middleware/data_source_queries.py | 21 ++++--- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/app_test.py b/app_test.py index e6042914..97d00baf 100644 --- a/app_test.py +++ b/app_test.py @@ -8,6 +8,7 @@ INSERT_LOG_QUERY, ) from middleware.data_source_queries import ( + data_sources_query, data_source_by_id_query, data_source_by_id_results, APPROVED_COLUMNS, @@ -19,6 +20,36 @@ current_datetime = datetime.datetime.now() DATETIME_STRING = current_datetime.strftime("%Y-%m-%d %H:%M:%S") +DATA_SOURCES_QUERY_RESULTS = [ + ( + "rec00T2YLS2jU7Tbn", + "Calls for Service for Chicago Police Department - IL", + None, + "Calls for Service", + "https://informationportal.igchicago.org/911-calls-for-cpd-service/", + None, + datetime.date(2019, 1, 1), + None, + True, + "Chicago Police Department - IL", + "Chicago", + "IL", + ), + ( + "recUGIoPQbJ6laBmr", + "311 Calls for City of Chicago", + "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", + "Calls for Service", + "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", + '["CSV", "XML", "RDF", "RSS"]', + datetime.date(2018, 12, 18), + None, + False, + "Chicago Police Department - IL", + "Chicago", + "IL", + ), + ] @pytest.fixture() def test_app(): @@ -230,7 +261,7 @@ def session(): # sql_query_log = f"INSERT INTO quick_search_query_logs (id, search, location, results, result_count, datetime_of_request, created_at) VALUES (1, 'test', 'test', '', 0, '{DATETIME_STRING}', '{DATETIME_STRING}')" # db_session.execute(sql_query_log) - yield db_session + yield connection connection.close() @@ -240,58 +271,35 @@ def session(): # unit tests def test_quick_search_queries(session): - session.execute(QUICK_SEARCH_TEST_SQL.format("calls", "chicago")) - results = session.fetchall() + cursor = session.cursor() + cursor.execute(QUICK_SEARCH_TEST_SQL.format("calls", "chicago")) + results = cursor.fetchall() assert len(results) > 0 results_str = json.dumps(results) print(INSERT_LOG_QUERY.format("calls", "chicago", results_str, 2, DATETIME_STRING)) - session.execute( + cursor.execute( INSERT_LOG_QUERY.format("calls", "chicago", results_str, 2, DATETIME_STRING) ) - session.execute( + cursor.execute( f"SELECT * FROM quick_search_query_logs WHERE datetime_of_request = '{DATETIME_STRING}'" ) - logs = session.fetchall() + logs = cursor.fetchall() assert len(logs) > 0 +def test_data_source_by_id_approved(session): + response = data_source_by_id_results(data_source_id="rec013MFNfBnrTpZj", conn=session) + + assert not response + + # quick-search def test_quicksearch_columns(): - query_results = [ - ( - "rec00T2YLS2jU7Tbn", - "Calls for Service for Chicago Police Department - IL", - None, - "Calls for Service", - "https://informationportal.igchicago.org/911-calls-for-cpd-service/", - None, - datetime.date(2019, 1, 1), - None, - True, - "Chicago Police Department - IL", - "Chicago", - "IL", - ), - ( - "recUGIoPQbJ6laBmr", - "311 Calls for City of Chicago", - "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", - "Calls for Service", - "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", - '["CSV", "XML", "RDF", "RSS"]', - datetime.date(2018, 12, 18), - None, - False, - "Chicago Police Department - IL", - "Chicago", - "IL", - ), - ] response = quick_search_query( - search="", location="", test_query_results=query_results + search="", location="", test_query_results=DATA_SOURCES_QUERY_RESULTS ) column_names = [ "airtable_uid", @@ -312,7 +320,7 @@ def test_quicksearch_columns(): # data-sources -def test_data_source_by_id_columns(client): +def test_data_source_by_id_columns(): query_results = ( "Calls for Service for Asheville Police Department - NC", None, @@ -420,22 +428,16 @@ def test_data_source_by_id_columns(client): assert not set(column_names).difference(response.keys()) -# def test_data_sources_approved(client): -# response = client.get("/data-sources", headers=HEADERS) +# def test_data_sources_approved(): +# response = data_sources_query(conn={}, test_query_results=DATA_SOURCES_QUERY_RESULTS) # unapproved_url = "https://joinstatepolice.ny.gov/15-mile-run" # assert ( -# len([d for d in response.json["data"] if d["source_url"] == unapproved_url]) +# len([d for d in response["data"] if d["source_url"] == unapproved_url]) # == 0 # ) -# def test_data_source_by_id_approved(client): -# response = client.get("/data-sources-by-id/rec013MFNfBnrTpZj", headers=HEADERS) - -# assert response.json == "Data source not found." - - # search-tokens diff --git a/middleware/data_source_queries.py b/middleware/data_source_queries.py index 593ea798..175110a0 100644 --- a/middleware/data_source_queries.py +++ b/middleware/data_source_queries.py @@ -87,7 +87,7 @@ def data_source_by_id_results(conn, data_source_id): joined_column_names = ", ".join(all_approved_columns) sql_query = """ SELECT - {} + {0} FROM agency_source_link INNER JOIN @@ -95,12 +95,10 @@ def data_source_by_id_results(conn, data_source_id): INNER JOIN agencies ON agency_source_link.agency_described_linked_uid = agencies.airtable_uid WHERE - data_sources.approval_status = 'approved' AND data_sources.airtable_uid = %s - """.format( - joined_column_names - ) + data_sources.approval_status = 'approved' AND data_sources.airtable_uid = '{1}' + """.format(joined_column_names, data_source_id) - cursor.execute(sql_query, (data_source_id,)) + cursor.execute(sql_query) return cursor.fetchone() @@ -126,7 +124,8 @@ def data_source_by_id_query(data_source_id="", test_query_results=[], conn={}): return data_source_details -def data_sources_query(conn): +def data_sources_results(conn): + cursor = conn.cursor() data_source_approved_columns = [ f"data_sources.{approved_column}" for approved_column in APPROVED_COLUMNS ] @@ -134,7 +133,6 @@ def data_sources_query(conn): joined_column_names = ", ".join(data_source_approved_columns) - cursor = conn.cursor() sql_query = """ SELECT {} @@ -150,7 +148,12 @@ def data_sources_query(conn): joined_column_names ) cursor.execute(sql_query) - results = cursor.fetchall() + + return cursor.fetchall() + + +def data_sources_query(conn={}, test_query_results=[]): + results = data_sources_results(conn, "", "") if conn else test_query_results data_source_output_columns = APPROVED_COLUMNS + ["agency_name"] From 7caa04c27178b13d2f2262bfbc138f8f0bf59bff Mon Sep 17 00:00:00 2001 From: Marty Bode Date: Fri, 12 Jan 2024 14:38:39 -0500 Subject: [PATCH 2/5] wip --- app_test.py | 51 ++++++++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/app_test.py b/app_test.py index 97d00baf..abad19f0 100644 --- a/app_test.py +++ b/app_test.py @@ -9,6 +9,7 @@ ) from middleware.data_source_queries import ( data_sources_query, + data_sources_results, data_source_by_id_query, data_source_by_id_results, APPROVED_COLUMNS, @@ -265,29 +266,23 @@ def session(): connection.close() -# @pytest.fixture -# def setup_db(session): +# unit tests +def test_data_sources(session): + response = data_sources_results(conn=session) + assert response -# unit tests -def test_quick_search_queries(session): - cursor = session.cursor() - cursor.execute(QUICK_SEARCH_TEST_SQL.format("calls", "chicago")) - results = cursor.fetchall() - assert len(results) > 0 +def test_data_sources_approved(session): + response = data_sources_results(conn=session) - results_str = json.dumps(results) - print(INSERT_LOG_QUERY.format("calls", "chicago", results_str, 2, DATETIME_STRING)) - cursor.execute( - INSERT_LOG_QUERY.format("calls", "chicago", results_str, 2, DATETIME_STRING) - ) - cursor.execute( - f"SELECT * FROM quick_search_query_logs WHERE datetime_of_request = '{DATETIME_STRING}'" - ) - logs = cursor.fetchall() + assert len([d for d in response if 'https://joinstatepolice.ny.gov/15-mile-run' in d]) == 0 - assert len(logs) > 0 + +def test_data_source_by_id_results(session): + response = data_source_by_id_results(data_source_id="rec00T2YLS2jU7Tbn", conn=session) + + assert response def test_data_source_by_id_approved(session): @@ -320,6 +315,13 @@ def test_quicksearch_columns(): # data-sources +def test_data_sources_columns(): + response = data_sources_query(conn={}, test_query_results=DATA_SOURCES_QUERY_RESULTS) + column_names = APPROVED_COLUMNS + ["agency_name"] + + assert not set(column_names).difference(response[0].keys()) + + def test_data_source_by_id_columns(): query_results = ( "Calls for Service for Asheville Police Department - NC", @@ -428,19 +430,6 @@ def test_data_source_by_id_columns(): assert not set(column_names).difference(response.keys()) -# def test_data_sources_approved(): -# response = data_sources_query(conn={}, test_query_results=DATA_SOURCES_QUERY_RESULTS) -# unapproved_url = "https://joinstatepolice.ny.gov/15-mile-run" - -# assert ( -# len([d for d in response["data"] if d["source_url"] == unapproved_url]) -# == 0 -# ) - - -# search-tokens - - # user From 27b6b21a3f3be006555519a33780a2cf599e3e88 Mon Sep 17 00:00:00 2001 From: Marty Bode Date: Thu, 18 Jan 2024 10:52:59 -0500 Subject: [PATCH 3/5] cleaned, fixed tests --- app_test.py | 445 +++++++----------------------- do_db_ddl_clean.sql | 6 + middleware/data_source_queries.py | 26 +- middleware/quick_search_query.py | 106 ++----- regular_api_checks.py | 4 +- resources/Archives.py | 71 ++--- 6 files changed, 165 insertions(+), 493 deletions(-) diff --git a/app_test.py b/app_test.py index abad19f0..bac08871 100644 --- a/app_test.py +++ b/app_test.py @@ -4,53 +4,37 @@ from middleware.quick_search_query import ( unaltered_search_query, quick_search_query, - QUICK_SEARCH_TEST_SQL, - INSERT_LOG_QUERY, + QUICK_SEARCH_COLUMNS, ) from middleware.data_source_queries import ( data_sources_query, data_sources_results, data_source_by_id_query, data_source_by_id_results, - APPROVED_COLUMNS, + DATA_SOURCES_APPROVED_COLUMNS, +) + +from middleware.archives_queries import ( + archives_get_results, + archives_get_query, + archives_put_broken_as_of_results, + archives_put_last_cached_results, + ARCHIVES_GET_COLUMNS, +) +from app_test_data import ( + DATA_SOURCES_ROWS, + DATA_SOURCE_QUERY_RESULTS, + QUICK_SEARCH_QUERY_RESULTS, + AGENCIES_ROWS, + DATA_SOURCES_ID_QUERY_RESULTS, + ARCHIVES_GET_QUERY_RESULTS, ) import datetime -import json import sqlite3 current_datetime = datetime.datetime.now() DATETIME_STRING = current_datetime.strftime("%Y-%m-%d %H:%M:%S") -DATA_SOURCES_QUERY_RESULTS = [ - ( - "rec00T2YLS2jU7Tbn", - "Calls for Service for Chicago Police Department - IL", - None, - "Calls for Service", - "https://informationportal.igchicago.org/911-calls-for-cpd-service/", - None, - datetime.date(2019, 1, 1), - None, - True, - "Chicago Police Department - IL", - "Chicago", - "IL", - ), - ( - "recUGIoPQbJ6laBmr", - "311 Calls for City of Chicago", - "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", - "Calls for Service", - "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", - '["CSV", "XML", "RDF", "RSS"]', - datetime.date(2018, 12, 18), - None, - False, - "Chicago Police Department - IL", - "Chicago", - "IL", - ), - ] @pytest.fixture() def test_app(): @@ -77,187 +61,21 @@ def session(): for query in sql_queries: db_session.execute(query.replace("\n", "")) - data_source_rows = [ - { - "airtable_uid": "rec00T2YLS2jU7Tbn", - "name": "Calls for Service for Chicago Police Department - IL", - "description": None, - "record_type": "Calls for Service", - "source_url": "https://informationportal.igchicago.org/911-calls-for-cpd-service/", - "record_format": None, - "coverage_start": "2019-01-01", - "coverage_end": None, - "agency_supplied": True, - "agency_name": "Chicago Police Department - IL", - "municipality": "Chicago", - "state_iso": "IL", - "url_status": "ok", - "approval_status": "approved", - }, - { - "airtable_uid": "recUGIoPQbJ6laBmr", - "name": "311 Calls for City of Chicago", - "description": "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", - "record_type": "Calls for Service", - "source_url": "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", - "record_format": '["CSV", "XML", "RDF", "RSS"]', - "coverage_start": "2018-12-18", - "coverage_end": None, - "agency_supplied": "False", - "agency_name": "Chicago Police Department - IL", - "municipality": "Chicago", - "state_iso": "IL", - "url_status": "ok", - "approval_status": "approved", - }, - { - "airtable_uid": "rec8zJuEOvhAZCfAD", - "name": "Pittsburgh Police Complaints and Disciplinary Actions 2013-2022", - "description": 'This news article contains several data tables. Most useful are the two near the bottom. "Pittsburgh Police Disciplinary Action Report data 2013-2022" and "Pittsburgh Office of Municipal Investigations police-related complaint data 2013-2022". Both are in paginated HTML tables and can be downloaded as a CSV by clicking a "Get the data" link.\n', - "record_type": "Complaints & Misconduct", - "source_url": "https://www.publicsource.org/pittsburgh-bureau-police-discipline-complaints-disciplinary-matrix-new-chief/", - "record_format": '["CSV", "HTML table"]', - "coverage_start": "2013-01-01", - "coverage_end": "2022-12-31", - "agency_supplied": "False", - "agency_name": "Pittsburgh Bureau of Police - PA", - "municipality": "Pittsburgh", - "state_iso": "PA", - "url_status": "ok", - "approval_status": "approved", - }, - { - "airtable_uid": "rec8gO2K86yk9mQIU", - "name": "Officer Involved Shootings for Philadelphia Police Department - PA", - "description": None, - "record_type": "Officer Involved Shootings", - "source_url": "https://www.phillypolice.com/ois/", - "record_format": None, - "coverage_start": "2015-01-01", - "coverage_end": None, - "agency_supplied": True, - "agency_name": "Philadelphia Police Department - PA", - "municipality": "Philadelphia", - "state_iso": "PA", - }, - ] - all_columns = APPROVED_COLUMNS + ["airtable_uid"] - for row in data_source_rows: - valid_row = {k: v for k, v in row.items() if k in all_columns} - clean_row = [r if r is not None else "" for r in valid_row.values()] - fully_clean_row = [r if r is not True else "True" for r in clean_row] + for row in DATA_SOURCES_ROWS: + # valid_row = {k: v for k, v in row.items() if k in all_columns} + # clean_row = [r if r is not None else "" for r in row] + fully_clean_row = [str(r) for r in row] fully_clean_row_str = "'" + "', '".join(fully_clean_row) + "'" - col_str = ", ".join(valid_row.keys()) - db_session.execute( - f"insert into data_sources ({col_str}) values ({fully_clean_row_str})" - ) - - db_session.execute( - "insert into agency_source_link (link_id, airtable_uid, agency_described_linked_uid) values (1, 'rec00T2YLS2jU7Tbn', 'recv9fMNEQTbVarj2')" - ) - db_session.execute( - "insert into agency_source_link (link_id, airtable_uid, agency_described_linked_uid) values (2, 'rec8zJuEOvhAZCfAD', 'recxUlLdt3Wwov6P1')" - ) - db_session.execute( - "insert into agency_source_link (link_id, airtable_uid, agency_described_linked_uid) values (3, 'recUGIoPQbJ6laBmr', 'recv9fMNEQTbVarj2')" - ) + db_session.execute(f"insert into data_sources values ({fully_clean_row_str})") db_session.execute( - "insert into agency_source_link (link_id, airtable_uid, agency_described_linked_uid) values (4, 'rec8gO2K86yk9mQIU', 'recRvBpZqXM8mjddz')" + "update data_sources set broken_source_url_as_of = null where broken_source_url_as_of = 'NULL'" ) - agencies_rows = [ - ( - "Chicago Police Department - IL", - "Chicago Police Department", - "https://home.chicagopolice.org/", - "local", - "IL", - "Chicago", - "17031", - '["Cook"]', - 41.85861, - -87.62802, - None, - "recv9fMNEQTbVarj2", - 21, - "law enforcement/police", - None, - "60616", - '["recs7gYAGAMWEhLYC", "recyR4IVgqyIekzYB", "recBMxu4UcHCqjsWQ", "recqXzWtmhfV9z8Av", "recc5QWeCMGL17Ab6", "recD9bNxneanhBBDH", "recPUGM4OEopoRAXB", "rec10Mts7CmMPzcnx", "recTj2zTdu7Jp8oX9", "rec00T2YLS2jU7Tbn", "recpOsfsTV4bKmVfa", "recipwTdMN0jRADrt", "recjFk7X0KofjTPGK", "rec0Lqwrj6blSbkuz", "recYyAY8XVQJJVde6", "recgwPuRTwjAMUZeO", "recOLt1yQGlgrO5jQ", "recdAte1fo5FBruE7", "recUGIoPQbJ6laBmr", "recTw75rItd827L0r", "rechI06qD4od759xT", "recodGxvuxiTXPnOL"]', - None, - datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), - datetime.date(2023, 4, 6), - True, - None, - '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', - None, - datetime.datetime(2022, 8, 18, 18, 50, 7, tzinfo=datetime.timezone.utc), - "recuhY2ud60V41j0w", - ), - ( - "Pittsburgh Bureau of Police - PA", - "Pittsburgh Bureau of Police", - "http://pittsburghpa.gov/police/", - "local", - "PA", - "Pittsburgh", - "42003", - '["Allegheny"]', - 40.450523, - -80.02128, - None, - "recxUlLdt3Wwov6P1", - 46, - "law enforcement/police", - None, - "15233", - '["recJK8P5rWlLjSzgc", "recCkota2A2S7Z33q", "recAD4tPHp4IndO2c", "recIwxfj2Ko77ySMD", "recF3bBivp59xdVBW", "recwj8eU8vdTNSEEu", "rec4G2iyEb1UiYfh2", "recJOzE2fe0Srdn4X", "recOlNMNivWF9sumN", "recEFwKevbY7P5DPS", "recvfbfIwGJeKH1OB", "recCvmgUInsKZpP3k", "recrm2fG7gztK7Tfg", "recwHAhLNsz52XvqX", "recX0ez0i7fcDQDx1", "recuvx89h9QZpRSZV", "recEFNJB8aOIF7ucx", "rec8ILhFGC9694CMS", "rec3Oc64eiYe0Cphx", "recLsGQ6yBEvJXTc3", "recRIejFKCgQgsX3l", "recd4qlLMmoLb4Rds", "recmhO1J5gh9pzgUP", "recJ46NmxuyjkoonW", "recmMakoz1eKO6rdC", "recAGF2VxFsOSZHqb", "recBSLCZXuVj1Zy0R", "recV3HCVPrqP31sqp", "rec1LIccYrPQVAdgL", "recgyEBNyh7VNyrAH", "recBjOdBK3XuSCTZy", "recUIXIJKuWleztqy", "recw9GbEslgN5w6zt", "recSQKpHfaj15B249", "recjeyUxVgQh2gqUj", "recbv6UOuUesjrXgI", "rec5E9yiFbuY3dWEc", "recFcv4IvAkcrlTdF", "recP5qX2qNlsuv7Np", "reckDUGoOgKx3yjqf", "rectnkXtlHLV26ZQP", "recrVIpgDHsFbB8Jn", "receW1cbs1sDMQvRl", "recQY7IEhVIIFL7wv", "recsIealZCldEKOlX", "rec7OiCCI6XxDy6ti", "rec2doMHsYVl5i6Y7", "rec8zJuEOvhAZCfAD", "recORxPfDzvYoBO4E", "recwDqMgD47XqrkbK", "rec40yyKK0f5lua4Z"]', - None, - datetime.datetime(2023, 3, 23, 18, 0, 3, tzinfo=datetime.timezone.utc), - datetime.date(2023, 11, 3), - True, - None, - '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', - None, - datetime.datetime(2022, 8, 18, 18, 49, 27, tzinfo=datetime.timezone.utc), - "recACF0SHugE9icVH", - ), - ( - "Philadelphia Police Department - PA", - "Philadelphia Police Department", - "https://www.phillypolice.com/districts/22nd/index.html", - "local", - "PA", - "Philadelphia", - "42101", - '["Philadelphia"]', - 39.980556, - -75.16209, - None, - "recRvBpZqXM8mjddz", - 17, - "law enforcement/police", - None, - "19121", - '["recXlFbG4J6pkBdKn", "recHd6j7LT6iMWWzf", "rec7IloZhaAsjpGwt", "recnKS6MwvODb4okj", "recgsrvT60fwJK2zv", "rec8gO2K86yk9mQIU", "recsvkVLs3NHh6fEk", "recIWjgyMq9umFWdv", "recIZvrJ1JJQddfGP", "recVcjF6jZJ7bkujw", "rec8r19ChbuSwpSpJ", "recPSQHatnNFV7H29", "recNH6V5a4TpINhMj", "recwbcrNqdutfgrE7", "rec5RlBXnQOEpGJQz", "recAbsBlvWEsCifvz", "recanjxF6Ph3SNA5P", "recX4kQbeRFch59DU", "recpqSEq8bYcNvIv9", "recVyLpoO48utQuq6"]', - None, - datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), - datetime.date(2023, 5, 2), - True, - None, - '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', - None, - datetime.datetime(2022, 8, 18, 18, 50, 49, tzinfo=datetime.timezone.utc), - "rec6tZ0VTIMmKXCkH", - ), - ] - clean_row = [r if r is not None else "" for r in agencies_rows[0]] - fully_clean_row = [str(r) for r in clean_row] - fully_clean_row_str = "'" + "', '".join(fully_clean_row) + "'" - - db_session.execute(f"insert into agencies values ({fully_clean_row_str})") - db_session.execute(f"insert into state_names values (1, 'IL', 'Illinois')") - db_session.execute(f"insert into state_names values (2, 'PA', 'Pennsylvania')") + for row in AGENCIES_ROWS: + clean_row = [r if r is not None else "" for r in row] + fully_clean_row = [str(r) for r in clean_row] + fully_clean_row_str = "'" + "', '".join(fully_clean_row) + "'" + db_session.execute(f"insert into agencies values ({fully_clean_row_str})") # sql_query_log = f"INSERT INTO quick_search_query_logs (id, search, location, results, result_count, datetime_of_request, created_at) VALUES (1, 'test', 'test', '', 0, '{DATETIME_STRING}', '{DATETIME_STRING}')" # db_session.execute(sql_query_log) @@ -267,6 +85,12 @@ def session(): # unit tests +def test_unaltered_search_query(session): + response = unaltered_search_query(session.cursor(), "calls", "chicago") + + assert response + + def test_data_sources(session): response = data_sources_results(conn=session) @@ -276,158 +100,83 @@ def test_data_sources(session): def test_data_sources_approved(session): response = data_sources_results(conn=session) - assert len([d for d in response if 'https://joinstatepolice.ny.gov/15-mile-run' in d]) == 0 + assert ( + len([d for d in response if "https://joinstatepolice.ny.gov/15-mile-run" in d]) + == 0 + ) def test_data_source_by_id_results(session): - response = data_source_by_id_results(data_source_id="rec00T2YLS2jU7Tbn", conn=session) + response = data_source_by_id_results( + data_source_id="rec00T2YLS2jU7Tbn", conn=session + ) assert response def test_data_source_by_id_approved(session): - response = data_source_by_id_results(data_source_id="rec013MFNfBnrTpZj", conn=session) + response = data_source_by_id_results( + data_source_id="rec013MFNfBnrTpZj", conn=session + ) assert not response +def test_archives_get_results(session): + response = archives_get_results(conn=session) + + assert response + + +def test_archives_put_broken_as_of(session): + archives_put_broken_as_of_results( + id="rec00T2YLS2jU7Tbn", + broken_as_of=DATETIME_STRING, + last_cached=DATETIME_STRING, + conn=session, + ) + curs = session.cursor() + broken_check, last_check = curs.execute( + f"SELECT broken_source_url_as_of, last_cached FROM data_sources WHERE airtable_uid = 'rec00T2YLS2jU7Tbn'" + ).fetchone() + + assert broken_check == DATETIME_STRING + assert last_check == DATETIME_STRING + + +def test_archives_put_last_cached(session): + archives_put_last_cached_results( + id="recUGIoPQbJ6laBmr", last_cached=DATETIME_STRING, conn=session + ) + curs = session.cursor() + last_check = curs.execute( + f"SELECT last_cached FROM data_sources WHERE airtable_uid = 'recUGIoPQbJ6laBmr'" + ).fetchone()[0] + + assert last_check == DATETIME_STRING + + # quick-search def test_quicksearch_columns(): response = quick_search_query( - search="", location="", test_query_results=DATA_SOURCES_QUERY_RESULTS + search="", location="", test_query_results=QUICK_SEARCH_QUERY_RESULTS ) - column_names = [ - "airtable_uid", - "data_source_name", - "record_type", - "source_url", - "record_format", - "coverage_start", - "coverage_end", - "agency_name", - "municipality", - "state_iso", - ] - print(response["data"]) - - assert not set(column_names).difference(response["data"][0].keys()) + + assert not set(QUICK_SEARCH_COLUMNS).difference(response["data"][0].keys()) assert type(response["data"][1]["record_format"]) == list # data-sources def test_data_sources_columns(): - response = data_sources_query(conn={}, test_query_results=DATA_SOURCES_QUERY_RESULTS) - column_names = APPROVED_COLUMNS + ["agency_name"] - - assert not set(column_names).difference(response[0].keys()) + response = data_sources_query(conn={}, test_query_results=DATA_SOURCE_QUERY_RESULTS) + + assert not set(DATA_SOURCES_APPROVED_COLUMNS).difference(response[0].keys()) def test_data_source_by_id_columns(): - query_results = ( - "Calls for Service for Asheville Police Department - NC", - None, - None, - "Calls for Service", - "https://services.arcgis.com/aJ16ENn1AaqdFlqx/arcgis/rest/services/APD_CAD_911_Calls_2006/FeatureServer/0", - True, - None, - None, - None, - None, - datetime.date(2006, 1, 1), - datetime.date(2006, 12, 31), - None, - None, - None, - None, - None, - '["API", "Download"]', - "ArcGIS", - '["GIS / Shapefile"]', - None, - None, - None, - "https://docs.google.com/document/d/143a0LoGwNwmmHxJu1msxjOFAfAXPk7otQSWkrLtUDk0/edit?usp=sharing", - "https://pypi.org/project/openpolicedata/", - datetime.datetime(2023, 3, 2, 18, 36, 27, tzinfo=datetime.timezone.utc), - datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), - "ok", - None, - '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', - None, - None, - "approved", - None, - None, - None, - None, - None, - None, - None, - "https://www.ashevillenc.gov/department/police", - 18, - "law enforcement/police", - None, - "Asheville Police Department", - "local", - "NC", - "Asheville", - "28801", - "37021", - '["Buncombe"]', - 35.594677, - -82.54986, - '["recpWxJ9JVa6BtLi5", "reczwxaH31Wf9gRjS", "recBd7NrWsvfTyDk0", "recy24QB6I8FCVrQr", "recW6aQGuNyzedIDl", "recmrXPvQn9Gtfpba", "recsojoTxJ3g08qKl", "recTUW1QUZpsGVxoJ", "reckqhpMEvDgiDGXF", "recjnLeosesVTaW2r", "reccrwbvTL6Ttd8XT", "reckiy2nuY5iRptBm", "recyJMD6eYF9Ln98B", "recLdzmQMXC3XuPWU", "recRNvjKBo5LkUBS9", "recKFoiPMOmWFZvqM", "recordd7DcM3raQF7", "recV35fyFlof4pQXP"]', - None, - datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), - datetime.date(2023, 3, 2), - True, - None, - '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', - datetime.datetime(2022, 8, 18, 18, 50, 38, tzinfo=datetime.timezone.utc), - "recrCy8hHYuxC8ZhU", - None, - "reczwxaH31Wf9gRjS", - "recJDGmbd7UMFcfa0", - "Asheville Police Department - NC", - ) - response = data_source_by_id_query("", query_results, {}) - column_names = [ - "description", - "record_type", - "agency_name", - "state_iso", - "county_name", - "municipality", - "agency_type", - "jurisdiction_type", - "source_url", - "readme_url", - "access_type", - "record_format", - "detail_level", - "size", - "access_type", - "access_notes", - "records_not_online", - "agency_supplied", - "supplying_entity", - "agency_originated", - "originating_entity", - "coverage_start", - "coverage_end", - "source_last_updated", - "update_frequency", - "update_method", - "retention_schedule", - "number_of_records_available", - "scraper_url", - "data_source_created", - "data_source_id", - "agency_id", - ] - - assert not set(column_names).difference(response.keys()) + response = data_source_by_id_query("", DATA_SOURCES_ID_QUERY_RESULTS, {}) + + assert not set(DATA_SOURCES_APPROVED_COLUMNS).difference(response.keys()) # user @@ -447,18 +196,12 @@ def test_data_source_by_id_columns(): # archives -# def test_get_archives_columns(client): -# response = client.get("/archives", headers=HEADERS) - -# column_names = [ -# "id", -# "source_url", -# "update_frequency", -# "last_cached", -# "agency_name", -# ] - -# assert not set(column_names).difference(response.json[0].keys()) +def test_archives_get_columns(): + response = archives_get_query( + test_query_results=ARCHIVES_GET_QUERY_RESULTS, conn={} + ) + + assert not set(ARCHIVES_GET_COLUMNS).difference(response[0].keys()) # agencies diff --git a/do_db_ddl_clean.sql b/do_db_ddl_clean.sql index 2bf49037..714215ce 100644 --- a/do_db_ddl_clean.sql +++ b/do_db_ddl_clean.sql @@ -158,3 +158,9 @@ CREATE TABLE if not exists volunteers ( created timestamp without time zone NOT NULL ); +INSERT INTO agency_source_link (link_id, airtable_uid, agency_described_linked_uid) VALUES (1, 'rec00T2YLS2jU7Tbn', 'recv9fMNEQTbVarj2'); +INSERT INTO agency_source_link (link_id, airtable_uid, agency_described_linked_uid) VALUES (2, 'rec8zJuEOvhAZCfAD', 'recxUlLdt3Wwov6P1'); +INSERT INTO agency_source_link (link_id, airtable_uid, agency_described_linked_uid) VALUES (3, 'recUGIoPQbJ6laBmr', 'recv9fMNEQTbVarj2'); +INSERT INTO agency_source_link (link_id, airtable_uid, agency_described_linked_uid) VALUES (4, 'rec8gO2K86yk9mQIU', 'recRvBpZqXM8mjddz'); +INSERT INTO state_names VALUES (1, 'IL', 'Illinois'); +INSERT INTO state_names VALUES (2, 'PA', 'Pennsylvania'); \ No newline at end of file diff --git a/middleware/data_source_queries.py b/middleware/data_source_queries.py index 175110a0..febd382b 100644 --- a/middleware/data_source_queries.py +++ b/middleware/data_source_queries.py @@ -1,6 +1,6 @@ from utilities.common import convert_dates_to_strings, format_arrays -APPROVED_COLUMNS = [ +DATA_SOURCES_APPROVED_COLUMNS = [ "name", "submitted_name", "description", @@ -74,7 +74,8 @@ def data_source_by_id_results(conn, data_source_id): cursor = conn.cursor() data_source_approved_columns = [ - f"data_sources.{approved_column}" for approved_column in APPROVED_COLUMNS + f"data_sources.{approved_column}" + for approved_column in DATA_SOURCES_APPROVED_COLUMNS ] agencies_approved_columns = [ f"agencies.{field}" for field in AGENCY_APPROVED_COLUMNS @@ -96,11 +97,15 @@ def data_source_by_id_results(conn, data_source_id): agencies ON agency_source_link.agency_described_linked_uid = agencies.airtable_uid WHERE data_sources.approval_status = 'approved' AND data_sources.airtable_uid = '{1}' - """.format(joined_column_names, data_source_id) + """.format( + joined_column_names, data_source_id + ) cursor.execute(sql_query) + result = cursor.fetchone() + cursor.close() - return cursor.fetchone() + return result def data_source_by_id_query(data_source_id="", test_query_results=[], conn={}): @@ -110,7 +115,9 @@ def data_source_by_id_query(data_source_id="", test_query_results=[], conn={}): result = test_query_results if result: - data_source_and_agency_columns = APPROVED_COLUMNS + AGENCY_APPROVED_COLUMNS + data_source_and_agency_columns = ( + DATA_SOURCES_APPROVED_COLUMNS + AGENCY_APPROVED_COLUMNS + ) data_source_and_agency_columns.append("data_source_id") data_source_and_agency_columns.append("agency_id") data_source_and_agency_columns.append("agency_name") @@ -127,7 +134,8 @@ def data_source_by_id_query(data_source_id="", test_query_results=[], conn={}): def data_sources_results(conn): cursor = conn.cursor() data_source_approved_columns = [ - f"data_sources.{approved_column}" for approved_column in APPROVED_COLUMNS + f"data_sources.{approved_column}" + for approved_column in DATA_SOURCES_APPROVED_COLUMNS ] data_source_approved_columns.append("agencies.name as agency_name") @@ -148,14 +156,16 @@ def data_sources_results(conn): joined_column_names ) cursor.execute(sql_query) + results = cursor.fetchall() + cursor.close() - return cursor.fetchall() + return results def data_sources_query(conn={}, test_query_results=[]): results = data_sources_results(conn, "", "") if conn else test_query_results - data_source_output_columns = APPROVED_COLUMNS + ["agency_name"] + data_source_output_columns = DATA_SOURCES_APPROVED_COLUMNS + ["agency_name"] data_source_matches = [ dict(zip(data_source_output_columns, result)) for result in results diff --git a/middleware/quick_search_query.py b/middleware/quick_search_query.py index d6bce950..8029336b 100644 --- a/middleware/quick_search_query.py +++ b/middleware/quick_search_query.py @@ -3,39 +3,22 @@ import datetime from utilities.common import convert_dates_to_strings, format_arrays -QUICK_SEARCH_SQL = """ - SELECT - data_sources.airtable_uid, - data_sources.name AS data_source_name, - data_sources.description, - data_sources.record_type, - data_sources.source_url, - data_sources.record_format, - data_sources.coverage_start, - data_sources.coverage_end, - data_sources.agency_supplied, - agencies.name AS agency_name, - agencies.municipality, - agencies.state_iso - FROM - agency_source_link - INNER JOIN - data_sources ON agency_source_link.airtable_uid = data_sources.airtable_uid - INNER JOIN - agencies ON agency_source_link.agency_described_linked_uid = agencies.airtable_uid - INNER JOIN - state_names ON agencies.state_iso = state_names.state_iso - WHERE - (data_sources.name ILIKE %s OR data_sources.description ILIKE %s OR data_sources.record_type ILIKE %s OR data_sources.tags ILIKE %s) - AND (agencies.county_name ILIKE %s OR substr(agencies.county_name,3,length(agencies.county_name)-4) || ' county' ILIKE %s - OR agencies.state_iso ILIKE %s OR agencies.municipality ILIKE %s OR agencies.agency_type ILIKE %s OR agencies.jurisdiction_type ILIKE %s - OR agencies.name ILIKE %s OR state_names.state_name ILIKE %s) - AND data_sources.approval_status = 'approved' - AND data_sources.url_status not in ('broken', 'none found') - -""" +QUICK_SEARCH_COLUMNS = [ + "airtable_uid", + "data_source_name", + "description", + "record_type", + "source_url", + "record_format", + "coverage_start", + "coverage_end", + "agency_supplied", + "agency_name", + "municipality", + "state_iso", +] -QUICK_SEARCH_TEST_SQL = """ +QUICK_SEARCH_SQL = """ SELECT data_sources.airtable_uid, data_sources.name AS data_source_name, @@ -72,25 +55,11 @@ def unaltered_search_query(cursor, search, location): print(f"Query parameters: '%{search}%', '%{location}%'") - cursor.execute( - QUICK_SEARCH_SQL, - ( - f"%{search}%", - f"%{search}%", - f"%{search}%", - f"%{search}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - ), - ) + cursor.execute(QUICK_SEARCH_SQL.format(search.title(), location.title())) + results = cursor.fetchall() + cursor.close() - return cursor.fetchall() + return results def spacy_search_query(cursor, search, location): @@ -105,24 +74,12 @@ def spacy_search_query(cursor, search, location): print(f"Query parameters: '%{depluralized_search_term}%', '%{location}%'") cursor.execute( - QUICK_SEARCH_SQL, - ( - f"%{depluralized_search_term}%", - f"%{depluralized_search_term}%", - f"%{depluralized_search_term}%", - f"%{depluralized_search_term}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - f"%{location}%", - ), + QUICK_SEARCH_SQL.format(depluralized_search_term.title(), location.title()) ) + results = cursor.fetchall() + cursor.close() - return cursor.fetchall() + return results def quick_search_query(search="", location="", test_query_results=[], conn={}): @@ -154,22 +111,9 @@ def quick_search_query(search="", location="", test_query_results=[], conn={}): else unaltered_results ) - column_names = [ - "airtable_uid", - "data_source_name", - "description", - "record_type", - "source_url", - "record_format", - "coverage_start", - "coverage_end", - "agency_supplied", - "agency_name", - "municipality", - "state_iso", + data_source_matches = [ + dict(zip(QUICK_SEARCH_COLUMNS, result)) for result in results ] - - data_source_matches = [dict(zip(column_names, result)) for result in results] data_source_matches_converted = [] for data_source_match in data_source_matches: data_source_match = convert_dates_to_strings(data_source_match) diff --git a/regular_api_checks.py b/regular_api_checks.py index bf4030a4..35b30dc0 100644 --- a/regular_api_checks.py +++ b/regular_api_checks.py @@ -152,7 +152,7 @@ def test_put_archives(): headers=HEADERS, json=json.dumps( { - "id": "test", + "id": "45a4cd5d-26da-473a-a98e-a39fbcf4a96c", "last_cached": datetime_string, "broken_source_url_as_of": "", } @@ -170,7 +170,7 @@ def test_put_archives_brokenasof(): headers=HEADERS, json=json.dumps( { - "id": "test", + "id": "45a4cd5d-26da-473a-a98e-a39fbcf4a96c", "last_cached": datetime_string, "broken_source_url_as_of": datetime_string, } diff --git a/resources/Archives.py b/resources/Archives.py index 7a50c611..37509ac9 100644 --- a/resources/Archives.py +++ b/resources/Archives.py @@ -1,6 +1,7 @@ from middleware.security import api_required +from middleware.archives_queries import archives_get_query, archives_put_query from flask_restful import Resource, request -from utilities.common import convert_dates_to_strings + import json @@ -11,40 +12,11 @@ def __init__(self, **kwargs): @api_required def get(self): try: - cursor = self.psycopg2_connection.cursor() - sql_query = """ - SELECT - data_sources.airtable_uid, - data_sources.source_url, - data_sources.update_frequency, - data_sources.last_cached, - agencies.name - FROM - agency_source_link - INNER JOIN - data_sources ON agency_source_link.airtable_uid = data_sources.airtable_uid - INNER JOIN - agencies ON agency_source_link.agency_described_linked_uid = agencies.airtable_uid - WHERE - (data_sources.last_cached IS NULL OR data_sources.update_frequency IS NOT NULL) AND data_sources.broken_source_url_as_of IS NULL AND data_sources.source_url IS NOT NULL - """ - cursor.execute(sql_query) - results = cursor.fetchall() - - column_names = [ - "id", - "source_url", - "update_frequency", - "last_cached", - "agency_name", - ] - - archive_results = [dict(zip(column_names, result)) for result in results] - - for item in archive_results: - convert_dates_to_strings(item) + archives_combined_results_clean = archives_get_query( + test_query_results=[], conn=self.psycopg2_connection + ) - return archive_results + return archives_combined_results_clean except Exception as e: self.psycopg2_connection.rollback() @@ -56,23 +28,20 @@ def put(self): try: json_data = request.get_json() data = json.loads(json_data) - - cursor = self.psycopg2_connection.cursor() - - if data["broken_source_url_as_of"]: - sql_query = "UPDATE data_sources SET broken_source_url_as_of = %s, last_cached = %s WHERE airtable_uid = %s" - cursor.execute( - sql_query, - (data["broken_source_url_as_of"], data["last_cached"], data["id"]), - ) - else: - sql_query = ( - "UPDATE data_sources SET last_cached = %s WHERE airtable_uid = %s" - ) - cursor.execute(sql_query, (data["last_cached"], data["id"])) - - self.psycopg2_connection.commit() - cursor.close() + id = data["id"] if "id" in data else None + broken_as_of = ( + data["broken_source_url_as_of"] + if "broken_source_url_as_of" in data + else None + ) + last_cached = data["last_cached"] if "last_cached" in data else None + + archives_put_query( + id=id, + broken_as_of=broken_as_of, + last_cached=last_cached, + conn=self.psycopg2_connection, + ) return {"status": "success"} From b5227b55929c2aa5977b569c992add3e38de99fc Mon Sep 17 00:00:00 2001 From: Marty Bode Date: Thu, 18 Jan 2024 10:56:50 -0500 Subject: [PATCH 4/5] missing file --- middleware/archives_queries.py | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 middleware/archives_queries.py diff --git a/middleware/archives_queries.py b/middleware/archives_queries.py new file mode 100644 index 00000000..f9197cfc --- /dev/null +++ b/middleware/archives_queries.py @@ -0,0 +1,70 @@ +from utilities.common import convert_dates_to_strings + +ARCHIVES_GET_COLUMNS = [ + "id", + "source_url", + "update_frequency", + "last_cached", + "agency_name", +] + + +def archives_get_results(conn): + cursor = conn.cursor() + sql_query = """ + SELECT + data_sources.airtable_uid, + data_sources.source_url, + data_sources.update_frequency, + data_sources.last_cached, + data_sources.broken_source_url_as_of, + agencies.name + FROM + agency_source_link + INNER JOIN + data_sources ON agency_source_link.airtable_uid = data_sources.airtable_uid + INNER JOIN + agencies ON agency_source_link.agency_described_linked_uid = agencies.airtable_uid + WHERE + (data_sources.last_cached IS NULL OR data_sources.update_frequency IS NOT NULL) AND data_sources.broken_source_url_as_of IS NULL AND data_sources.source_url IS NOT NULL + """ + cursor.execute(sql_query) + + return cursor.fetchall() + + +def archives_get_query(test_query_results=[], conn={}): + results = ( + archives_get_results(conn) if not test_query_results else test_query_results + ) + archives_combined_results = [ + dict(zip(ARCHIVES_GET_COLUMNS, result)) for result in results + ] + archives_combined_results_clean = [] + for item in archives_combined_results: + archives_combined_results_clean.append(convert_dates_to_strings(item)) + + return archives_combined_results_clean + + +def archives_put_broken_as_of_results(id, broken_as_of, last_cached, conn): + cursor = conn.cursor() + sql_query = "UPDATE data_sources SET broken_source_url_as_of = '{0}', last_cached = '{1}' WHERE airtable_uid = '{2}'" + cursor.execute(sql_query.format(broken_as_of, last_cached, id)) + cursor.close() + + +def archives_put_last_cached_results(id, last_cached, conn): + cursor = conn.cursor() + sql_query = "UPDATE data_sources SET last_cached = '{0}' WHERE airtable_uid = '{1}'" + cursor.execute(sql_query.format(last_cached, id)) + cursor.close() + + +def archives_put_query(id="", broken_as_of="", last_cached="", conn={}): + if broken_as_of: + archives_put_broken_as_of_results(id, broken_as_of, last_cached, conn) + else: + archives_put_last_cached_results(id, last_cached, conn) + + conn.commit() From 360f5f5183895ad25c64eb3cf2f916a68d13c6f7 Mon Sep 17 00:00:00 2001 From: Marty Bode Date: Thu, 18 Jan 2024 10:58:43 -0500 Subject: [PATCH 5/5] other missing file --- app_test_data.py | 397 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 app_test_data.py diff --git a/app_test_data.py b/app_test_data.py new file mode 100644 index 00000000..dd0b0035 --- /dev/null +++ b/app_test_data.py @@ -0,0 +1,397 @@ +import datetime + +DATA_SOURCES_ROWS = [ + ( + "Calls for Service for Chicago Police Department - IL", + "NULL", + "NULL", + "Calls for Service", + "https://informationportal.igchicago.org/911-calls-for-cpd-service/", + True, + "NULL", + "NULL", + "NULL", + datetime.date(2019, 1, 1), + "NULL", + "NULL", + "Summarized totals", + "NULL", + "NULL", + '["Web page"]', + "NULL", + "NULL", + "NULL", + "Bi-weekly", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "rec00T2YLS2jU7Tbn", + "NULL", + datetime.datetime(2022, 9, 28, 22, 25, 21, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), + "NULL", + "NULL", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + "Chicago city, Illinois", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "ok", + "approved", + ), + ( + "311 Calls for City of Chicago", + "311 Calls for City of Chicago", + "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", + "Calls for Service", + "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", + False, + "City of Chicago\n", + False, + "NULL", + datetime.date(2018, 12, 18), + "NULL", + datetime.date(2023, 2, 21), + "NULL", + "NULL", + "NULL", + '["Web page", "API", "Download"]', + "NULL", + "Opendata", + '["CSV", "XML", "RDF", "RSS"]', + "NULL", + "NULL", + "NULL", + "NULL", + "City of Chicago\n", + "NULL", + "recUGIoPQbJ6laBmr", + "https://github.com/fgregg/chicagorequests", + datetime.datetime(2023, 2, 21, 15, 19, 25, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), + "NULL", + "NULL", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + datetime.date(2023, 7, 12), + "NULL", + "ok", + "approved", + ), +] +DATA_SOURCE_QUERY_RESULTS = [ + ( + "Calls for Service for Chicago Police Department - IL", + "NULL", + "NULL", + "Calls for Service", + "https://informationportal.igchicago.org/911-calls-for-cpd-service/", + True, + "NULL", + "NULL", + "NULL", + "NULL", + datetime.date(2019, 1, 1), + "NULL", + "NULL", + "NULL", + "Summarized totals", + "NULL", + "NULL", + '["Web page"]', + "NULL", + "NULL", + "Bi-weekly", + "NULL", + "NULL", + "NULL", + "NULL", + datetime.datetime(2022, 9, 28, 22, 25, 21, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), + "ok", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "Chicago city, Illinois", + "NULL", + "approved", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "Chicago Police Department - IL", + ), + ( + "Officer Involved Shootings for St. Louis (city) Police Department - MO", + "NULL", + "NULL", + "Officer Involved Shootings", + "NULL", + True, + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + datetime.datetime(2022, 9, 28, 22, 25, 30, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), + "None found", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "St. Louis city, Missouri", + "NULL", + "approved", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "St. Louis (city) Police Department - MO", + ), +] +DATA_SOURCES_ID_QUERY_RESULTS = ( + "Calls for Service for Asheville Police Department - NC", + "NULL", + "NULL", + "Calls for Service", + "https://services.arcgis.com/aJ16ENn1AaqdFlqx/arcgis/rest/services/APD_CAD_911_Calls_2006/FeatureServer/0", + True, + "NULL", + "NULL", + "NULL", + "NULL", + datetime.date(2006, 1, 1), + datetime.date(2006, 12, 31), + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + '["API", "Download"]', + "ArcGIS", + '["GIS / Shapefile"]', + "NULL", + "NULL", + "NULL", + "https://docs.google.com/document/d/143a0LoGwNwmmHxJu1msxjOFAfAXPk7otQSWkrLtUDk0/edit?usp=sharing", + "https://pypi.org/project/openpolicedata/", + datetime.datetime(2023, 3, 2, 18, 36, 27, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 11, 8, 19, 6, 38, tzinfo=datetime.timezone.utc), + "ok", + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + "NULL", + "approved", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "NULL", + "https://www.ashevillenc.gov/department/police", + 18, + "law enforcement/police", + "NULL", + "Asheville Police Department", + "local", + "NC", + "Asheville", + "28801", + "37021", + '["Buncombe"]', + 35.594677, + -82.54986, + '["recpWxJ9JVa6BtLi5", "reczwxaH31Wf9gRjS", "recBd7NrWsvfTyDk0", "recy24QB6I8FCVrQr", "recW6aQGuNyzedIDl", "recmrXPvQn9Gtfpba", "recsojoTxJ3g08qKl", "recTUW1QUZpsGVxoJ", "reckqhpMEvDgiDGXF", "recjnLeosesVTaW2r", "reccrwbvTL6Ttd8XT", "reckiy2nuY5iRptBm", "recyJMD6eYF9Ln98B", "recLdzmQMXC3XuPWU", "recRNvjKBo5LkUBS9", "recKFoiPMOmWFZvqM", "recordd7DcM3raQF7", "recV35fyFlof4pQXP"]', + "NULL", + datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), + datetime.date(2023, 3, 2), + True, + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + datetime.datetime(2022, 8, 18, 18, 50, 38, tzinfo=datetime.timezone.utc), + "recrCy8hHYuxC8ZhU", + "NULL", + "reczwxaH31Wf9gRjS", + "recJDGmbd7UMFcfa0", + "Asheville Police Department - NC", +) + +QUICK_SEARCH_QUERY_RESULTS = [ + ( + "rec00T2YLS2jU7Tbn", + "Calls for Service for Chicago Police Department - IL", + "NULL", + "Calls for Service", + "https://informationportal.igchicago.org/911-calls-for-cpd-service/", + "NULL", + datetime.date(2019, 1, 1), + "NULL", + True, + "Chicago Police Department - IL", + "Chicago", + "IL", + ), + ( + "recUGIoPQbJ6laBmr", + "311 Calls for City of Chicago", + "311 Service Requests received by the City of Chicago. This dataset includes requests created after the launch of the new 311 system on 12/18/2018 and some records from the previous system, indicated in the LEGACY\\_RECORD column.\n\nIncluded as a Data Source because in some cities 311 calls lead to police response; that does not appear to be the case in Chicago.\n", + "Calls for Service", + "https://data.cityofchicago.org/Service-Requests/311-Service-Requests/v6vf-nfxy", + '["CSV", "XML", "RDF", "RSS"]', + datetime.date(2018, 12, 18), + "NULL", + False, + "Chicago Police Department - IL", + "Chicago", + "IL", + ), +] + +AGENCIES_ROWS = [ + ( + "Chicago Police Department - IL", + "Chicago Police Department", + "https://home.chicagopolice.org/", + "local", + "IL", + "Chicago", + "17031", + '["Cook"]', + 41.85861, + -87.62802, + "NULL", + "recv9fMNEQTbVarj2", + 21, + "law enforcement/police", + "NULL", + "60616", + '["recs7gYAGAMWEhLYC", "recyR4IVgqyIekzYB", "recBMxu4UcHCqjsWQ", "recqXzWtmhfV9z8Av", "recc5QWeCMGL17Ab6", "recD9bNxneanhBBDH", "recPUGM4OEopoRAXB", "rec10Mts7CmMPzcnx", "recTj2zTdu7Jp8oX9", "rec00T2YLS2jU7Tbn", "recpOsfsTV4bKmVfa", "recipwTdMN0jRADrt", "recjFk7X0KofjTPGK", "rec0Lqwrj6blSbkuz", "recYyAY8XVQJJVde6", "recgwPuRTwjAMUZeO", "recOLt1yQGlgrO5jQ", "recdAte1fo5FBruE7", "recUGIoPQbJ6laBmr", "recTw75rItd827L0r", "rechI06qD4od759xT", "recodGxvuxiTXPnOL"]', + "NULL", + datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), + datetime.date(2023, 4, 6), + True, + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + datetime.datetime(2022, 8, 18, 18, 50, 7, tzinfo=datetime.timezone.utc), + "recuhY2ud60V41j0w", + ), + ( + "Pittsburgh Bureau of Police - PA", + "Pittsburgh Bureau of Police", + "http://pittsburghpa.gov/police/", + "local", + "PA", + "Pittsburgh", + "42003", + '["Allegheny"]', + 40.450523, + -80.02128, + "NULL", + "recxUlLdt3Wwov6P1", + 46, + "law enforcement/police", + "NULL", + "15233", + '["recJK8P5rWlLjSzgc", "recCkota2A2S7Z33q", "recAD4tPHp4IndO2c", "recIwxfj2Ko77ySMD", "recF3bBivp59xdVBW", "recwj8eU8vdTNSEEu", "rec4G2iyEb1UiYfh2", "recJOzE2fe0Srdn4X", "recOlNMNivWF9sumN", "recEFwKevbY7P5DPS", "recvfbfIwGJeKH1OB", "recCvmgUInsKZpP3k", "recrm2fG7gztK7Tfg", "recwHAhLNsz52XvqX", "recX0ez0i7fcDQDx1", "recuvx89h9QZpRSZV", "recEFNJB8aOIF7ucx", "rec8ILhFGC9694CMS", "rec3Oc64eiYe0Cphx", "recLsGQ6yBEvJXTc3", "recRIejFKCgQgsX3l", "recd4qlLMmoLb4Rds", "recmhO1J5gh9pzgUP", "recJ46NmxuyjkoonW", "recmMakoz1eKO6rdC", "recAGF2VxFsOSZHqb", "recBSLCZXuVj1Zy0R", "recV3HCVPrqP31sqp", "rec1LIccYrPQVAdgL", "recgyEBNyh7VNyrAH", "recBjOdBK3XuSCTZy", "recUIXIJKuWleztqy", "recw9GbEslgN5w6zt", "recSQKpHfaj15B249", "recjeyUxVgQh2gqUj", "recbv6UOuUesjrXgI", "rec5E9yiFbuY3dWEc", "recFcv4IvAkcrlTdF", "recP5qX2qNlsuv7Np", "reckDUGoOgKx3yjqf", "rectnkXtlHLV26ZQP", "recrVIpgDHsFbB8Jn", "receW1cbs1sDMQvRl", "recQY7IEhVIIFL7wv", "recsIealZCldEKOlX", "rec7OiCCI6XxDy6ti", "rec2doMHsYVl5i6Y7", "rec8zJuEOvhAZCfAD", "recORxPfDzvYoBO4E", "recwDqMgD47XqrkbK", "rec40yyKK0f5lua4Z"]', + "NULL", + datetime.datetime(2023, 3, 23, 18, 0, 3, tzinfo=datetime.timezone.utc), + datetime.date(2023, 11, 3), + True, + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + datetime.datetime(2022, 8, 18, 18, 49, 27, tzinfo=datetime.timezone.utc), + "recACF0SHugE9icVH", + ), + ( + "Philadelphia Police Department - PA", + "Philadelphia Police Department", + "https://www.phillypolice.com/districts/22nd/index.html", + "local", + "PA", + "Philadelphia", + "42101", + '["Philadelphia"]', + 39.980556, + -75.16209, + "NULL", + "recRvBpZqXM8mjddz", + 17, + "law enforcement/police", + "NULL", + "19121", + '["recXlFbG4J6pkBdKn", "recHd6j7LT6iMWWzf", "rec7IloZhaAsjpGwt", "recnKS6MwvODb4okj", "recgsrvT60fwJK2zv", "rec8gO2K86yk9mQIU", "recsvkVLs3NHh6fEk", "recIWjgyMq9umFWdv", "recIZvrJ1JJQddfGP", "recVcjF6jZJ7bkujw", "rec8r19ChbuSwpSpJ", "recPSQHatnNFV7H29", "recNH6V5a4TpINhMj", "recwbcrNqdutfgrE7", "rec5RlBXnQOEpGJQz", "recAbsBlvWEsCifvz", "recanjxF6Ph3SNA5P", "recX4kQbeRFch59DU", "recpqSEq8bYcNvIv9", "recVyLpoO48utQuq6"]', + "NULL", + datetime.datetime(2023, 5, 16, 17, 37, 6, tzinfo=datetime.timezone.utc), + datetime.date(2023, 5, 2), + True, + "NULL", + '{"id": "usrtLIB4Vr3jTH8Ro", "email": "josh.chamberlain@pdap.io", "name": "Josh Chamberlain"}', + "NULL", + datetime.datetime(2022, 8, 18, 18, 50, 49, tzinfo=datetime.timezone.utc), + "rec6tZ0VTIMmKXCkH", + ), +] + +ARCHIVES_GET_QUERY_RESULTS = [ + ( + "rec00T2YLS2jU7Tbn", + "https://informationportal.igchicago.org/911-calls-for-cpd-service/", + "Bi-weekly", + "NULL", + "Chicago Police Department - IL", + ), + ( + "rec04V1oMa6Dxt0Sl", + "https://www.columbus.gov/police-annualreports/", + "Annually", + datetime.date(2023, 9, 16), + "Columbus Division of Police - OH", + ), +]