diff --git a/server/config/nl_page/topic_cache.json b/server/config/nl_page/topic_cache.json index 42c9a91662..106de8de1d 100644 --- a/server/config/nl_page/topic_cache.json +++ b/server/config/nl_page/topic_cache.json @@ -15422,7 +15422,6 @@ ], "relevantVariableList": [ "UnemploymentRate_Person", - "Count_Person_NonWorker", "UnemploymentRate_Person_Female", "UnemploymentRate_Person_Male" ], diff --git a/server/integration_tests/test_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json b/server/integration_tests/test_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json index 71605661ae..41a63fda94 100644 --- a/server/integration_tests/test_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json +++ b/server/integration_tests/test_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json @@ -54,7 +54,7 @@ "Parts": [ { "CosineScore": [ - 0.9257855415344238 + 0.9257859587669373 ], "QueryPart": "male population", "SV": [ @@ -64,7 +64,7 @@ { "CosineScore": [ 0.9296979904174805, - 0.884743332862854 + 0.8847434520721436 ], "QueryPart": "female population", "SV": [ @@ -123,7 +123,7 @@ }, { "CosineScore": [ - 0.917032778263092 + 0.9170326590538025 ], "QueryPart": "population female population", "SV": [ diff --git a/server/integration_tests/test_data/detection_api_multivar/numberofpoorhispanicwomenwithphd/debug_info.json b/server/integration_tests/test_data/detection_api_multivar/numberofpoorhispanicwomenwithphd/debug_info.json index 88e50545fd..6fdd118b27 100644 --- a/server/integration_tests/test_data/detection_api_multivar/numberofpoorhispanicwomenwithphd/debug_info.json +++ b/server/integration_tests/test_data/detection_api_multivar/numberofpoorhispanicwomenwithphd/debug_info.json @@ -57,17 +57,17 @@ "Parts": [ { "CosineScore": [ - 0.8084911108016968, - 0.8080331087112427, - 0.8044111728668213, - 0.7984197735786438, - 0.7966943979263306, - 0.7915846705436707, - 0.790185809135437, + 0.8084912300109863, + 0.8080331683158875, + 0.8044112920761108, + 0.7984198331832886, + 0.7966946959495544, + 0.791584849357605, + 0.7901858687400818, 0.7865886688232422, - 0.777317464351654, - 0.776214063167572, - 0.7629760503768921 + 0.7773175239562988, + 0.7762140035629272, + 0.7629759907722473 ], "QueryPart": "number poor hispanic", "SV": [ @@ -104,7 +104,7 @@ { "CosineScore": [ 0.8433718681335449, - 0.8265039324760437 + 0.8265039920806885 ], "QueryPart": "number poor hispanic women", "SV": [ @@ -114,11 +114,11 @@ }, { "CosineScore": [ - 0.7646968960762024, + 0.7646970152854919, 0.7476688027381897, - 0.7377066612243652, - 0.7362767457962036, - 0.7305883169174194 + 0.73770672082901, + 0.7362766265869141, + 0.7305881977081299 ], "QueryPart": "phd", "SV": [ @@ -137,7 +137,7 @@ "Parts": [ { "CosineScore": [ - 0.7890844941139221 + 0.7890845537185669 ], "QueryPart": "number poor", "SV": [ @@ -146,10 +146,10 @@ }, { "CosineScore": [ - 0.8151503801345825, - 0.7885993719100952, - 0.7775554060935974, - 0.7752052545547485 + 0.8151504993438721, + 0.7885995507240295, + 0.7775554656982422, + 0.7752053737640381 ], "QueryPart": "hispanic women phd", "SV": [ diff --git a/server/integration_tests/test_data/detection_api_multivar/showmetheimpactofclimatechangeondrought/debug_info.json b/server/integration_tests/test_data/detection_api_multivar/showmetheimpactofclimatechangeondrought/debug_info.json index a92bf6e5a3..f96fd5b516 100644 --- a/server/integration_tests/test_data/detection_api_multivar/showmetheimpactofclimatechangeondrought/debug_info.json +++ b/server/integration_tests/test_data/detection_api_multivar/showmetheimpactofclimatechangeondrought/debug_info.json @@ -41,7 +41,7 @@ "Parts": [ { "CosineScore": [ - 0.8787486553192139 + 0.8787485361099243 ], "QueryPart": "show climate change", "SV": [ @@ -50,7 +50,7 @@ }, { "CosineScore": [ - 1.0 + 0.9999998211860657 ], "QueryPart": "drought", "SV": [ @@ -65,10 +65,10 @@ "Parts": [ { "CosineScore": [ - 0.794236958026886, - 0.7929013967514038, - 0.7586168050765991, - 0.758395791053772 + 0.7942368984222412, + 0.7929012775421143, + 0.7586166858673096, + 0.7583957314491272 ], "QueryPart": "show climate", "SV": [ diff --git a/server/integration_tests/test_data/detection_api_undata_dev_idx/chart_config.json b/server/integration_tests/test_data/detection_api_undata_dev_idx/chart_config.json index e035b7c2a0..2293464b5b 100644 --- a/server/integration_tests/test_data/detection_api_undata_dev_idx/chart_config.json +++ b/server/integration_tests/test_data/detection_api_undata_dev_idx/chart_config.json @@ -10,7 +10,9 @@ "comparisonVariables": [], "context": {}, "debug": {}, - "entities": ["Earth"], + "entities": [ + "Earth" + ], "nonPlaceEntities": [], "properties": [], "sessionId": "007_999999999", @@ -37,4 +39,4 @@ "dc/topic/sdg_8.8.2", "dc/topic/ILOEMP2WAPRT" ] -} +} \ No newline at end of file diff --git a/server/integration_tests/test_data/international/query_3/chart_config.json b/server/integration_tests/test_data/international/query_3/chart_config.json index ffe2dda68e..1761359c1c 100644 --- a/server/integration_tests/test_data/international/query_3/chart_config.json +++ b/server/integration_tests/test_data/international/query_3/chart_config.json @@ -178,6 +178,34 @@ ], "title": "Unemployment Rate" }, + { + "columns": [ + { + "tiles": [ + { + "statVarKey": [ + "UnemploymentRate_Person_Female" + ], + "title": "Unemployment Rate of Females in United Kingdom", + "type": "LINE" + } + ] + }, + { + "tiles": [ + { + "description": "Unemployment Rate of Females in United Kingdom", + "statVarKey": [ + "UnemploymentRate_Person_Female" + ], + "title": "Unemployment Rate of Females in United Kingdom", + "type": "HIGHLIGHT" + } + ] + } + ], + "title": "Unemployment Rate of Females" + }, { "columns": [ { @@ -225,6 +253,10 @@ "name": "Unemployment Rate", "statVar": "UnemploymentRate_Person" }, + "UnemploymentRate_Person_Female": { + "name": "Unemployment Rate of Females", + "statVar": "UnemploymentRate_Person_Female" + }, "sdg/EN_MAT_DOMCMPG": { "name": "Domestic material consumption per unit of GDP", "statVar": "sdg/EN_MAT_DOMCMPG" @@ -620,25 +652,10 @@ }, "childTopics": [], "exploreMore": { - "sdg/EN_MAT_DOMCMPG": { - "sdg_product": [ - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF1", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF11", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF121", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF122", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF13", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF14", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF2", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF21", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF22", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF3", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF4", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF41", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF413", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF421", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF422", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF_AGG3A", - "sdg/EN_MAT_DOMCMPG.PRODUCT--MF_AGG3B" + "UnemploymentRate_Person_Female": { + "age": [ + "UnemploymentRate_Person_25To74Years_Female", + "UnemploymentRate_Person_Upto25Years_Female" ] } }, diff --git a/server/lib/nl/common/debug_utils.py b/server/lib/nl/common/debug_utils.py index 3201ae0221..ab1d2a2c9e 100644 --- a/server/lib/nl/common/debug_utils.py +++ b/server/lib/nl/common/debug_utils.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import cast, Dict +import logging +from typing import Dict from server.lib.nl.detection.types import ClassificationType from server.lib.nl.detection.types import Detection @@ -134,6 +135,9 @@ def result_with_debug_info(data_dict: Dict, status: str, for entity in query_detection.places_detected.entities_found: entities_found_formatted += f"(name: {entity.name}, dcid: {entity.dcid}); " + query_transformations = query_detection_debug_logs.get( + 'query_transformations', {}) + debug_info.update({ 'places_detected': query_detection.places_detected.query_places_mentioned, @@ -143,8 +147,13 @@ def result_with_debug_info(data_dict: Dict, status: str, query_detection.places_detected.query_entities_mentioned, 'entities_resolved': entities_found_formatted, + 'query_index_type': + query_transformations.get('sv_detection_query_index_type', ''), 'query_with_places_removed': query_detection.places_detected.query_without_place_substr, + 'query_with_stop_words_removal': + query_transformations.get('sv_detection_query_stop_words_removal', + ''), 'query_detection_debug_logs': query_detection_debug_logs, }) diff --git a/server/routes/redirects.py b/server/routes/redirects.py index 660397b2b0..ca37e39aec 100644 --- a/server/routes/redirects.py +++ b/server/routes/redirects.py @@ -115,8 +115,8 @@ def demo(): def load_redirects(name): - local_file = gcs.download_file(bucket=GLOBAL_CONFIG_BUCKET, - filename='redirects.json') + local_file = gcs.maybe_download( + gcs.make_path(GLOBAL_CONFIG_BUCKET, 'redirects.json')) with open(local_file) as fp: mapping = json.load(fp) return mapping.get(name, '/') diff --git a/static/css/explore.scss b/static/css/explore.scss index aa56d9a880..994328a890 100644 --- a/static/css/explore.scss +++ b/static/css/explore.scss @@ -323,6 +323,11 @@ width: 100%; } +.nl-query-result-debug-info .highlight { + color: blue; + font-weight: bold; +} + #ranking-table-container { margin-left: 24px; } diff --git a/static/js/apps/explore/debug_info.tsx b/static/js/apps/explore/debug_info.tsx index 4443e47e11..ac4ea40fbb 100644 --- a/static/js/apps/explore/debug_info.tsx +++ b/static/js/apps/explore/debug_info.tsx @@ -198,7 +198,9 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { entitiesResolved: props.debugData["entities_resolved"] || [], mainPlaceDCID: props.debugData["main_place_dcid"], mainPlaceName: props.debugData["main_place_name"], + queryIndexType: props.debugData["query_index_type"], queryWithoutPlaces: props.debugData["query_with_places_removed"], + queryWithoutStopWords: props.debugData["query_with_stop_words_removal"], queryDetectionDebugLogs: props.debugData["query_detection_debug_logs"], svScores: props.debugData["sv_matching"] || {}, svSentences: props.debugData["svs_to_sentences"], @@ -238,28 +240,40 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { - Execution Status: {debugInfo.status} + Execution Status: {" "} + {debugInfo.status} - Detection Type: {debugInfo.detectionType} + Detection Type: + {debugInfo.detectionType} Place Detection Type: {" "} - {debugInfo.placeDetectionType.toUpperCase()} + + {debugInfo.placeDetectionType.toUpperCase()} + - Original Query: {debugInfo.originalQuery} + Original Query: + {debugInfo.originalQuery} - Blocked: {debugInfo.blocked.toString()} + Blocked:{" "} + {debugInfo.blocked.toString()} + + + Query index type: + {debugInfo.queryIndexType} Query without places: - {debugInfo.queryWithoutPlaces} + {debugInfo.queryWithoutPlaces} - Query used for variable detection: - {debugInfo.svScores.Query || ""} + Query without stop words: + + {debugInfo.queryWithoutStopWords || ""} + Place Detection: @@ -267,18 +281,26 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { Places Detected:{" "} - {debugInfo.placesDetected - ? debugInfo.placesDetected.join(", ") - : ""} + + {" "} + {debugInfo.placesDetected + ? debugInfo.placesDetected.join(", ") + : ""} + - Places Resolved: {debugInfo.placesResolved} + + Places Resolved: + {debugInfo.placesResolved} + - Main Place: {debugInfo.mainPlaceName} (dcid:{" "} - {debugInfo.mainPlaceDCID}) + Main Place: + + {debugInfo.mainPlaceName} (dcid: {debugInfo.mainPlaceDCID}) + @@ -290,13 +312,16 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { - Entities Resolved: {debugInfo.entitiesResolved} + + Entities Resolved:{" "} + {debugInfo.entitiesResolved} + Query Type Detection: - Ranking classification: {debugInfo.rankingClassification} + Ranking classification: @@ -338,16 +363,6 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { Date classification: {debugInfo.dateClassification} - - Query Detection: - - - - - {JSON.stringify(debugInfo.queryDetectionDebugLogs, null, 2)} - - - Single Variables Matches: @@ -402,7 +417,16 @@ export function DebugInfo(props: DebugInfoProps): JSX.Element { )} - + + Query Detection: + + + + + {JSON.stringify(debugInfo.queryDetectionDebugLogs, null, 2)} + + + setIsCollapsed(!isCollapsed)}
- {JSON.stringify(debugInfo.queryDetectionDebugLogs, null, 2)} -
+ {JSON.stringify(debugInfo.queryDetectionDebugLogs, null, 2)} +