From 11f3a4e3eebdff906d7709d23f298a7d46bd52a5 Mon Sep 17 00:00:00 2001 From: Qifeng Date: Mon, 10 Jan 2022 09:50:31 +1100 Subject: [PATCH] #622 fix(urlencode lsid) --- .../ala/distribution/DistributionService.java | 11 +- .../distribution/DistributionServiceImpl.java | 5 + .../DistributionOutlierTransform.java | 13 +- .../pipelines/src/main/resources/logback.xml | 2 +- .../ala/outlier/DistributionOutlierTest.java | 323 +++++++++++++++++- 5 files changed, 338 insertions(+), 16 deletions(-) diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionService.java b/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionService.java index 7f46738044..34a6b443b4 100644 --- a/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionService.java +++ b/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionService.java @@ -12,16 +12,21 @@ public interface DistributionService { @GET("distribution/") Call> getLayers(); + /** + * @param id the species id MUST be URLEncoded + * @param nowkt + * @return + */ @GET("distribution/lsids/{id}") Call> getLayersByLsid( - @Path("id") String id, @Query("nowkt") String nowkt); + @Path(value = "id") String id, @Query("nowkt") String nowkt); /** - * @param lsid + * @param id the species id MUST be URLEncoded * @param points Map> * @return */ @POST("distribution/outliers/{id}") Call> outliers( - @Path("id") String lsid, @Body Map> points); + @Path(value = "id") String id, @Body Map> points); } diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionServiceImpl.java b/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionServiceImpl.java index e6d24d0f9e..c1700c1b7e 100644 --- a/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionServiceImpl.java +++ b/livingatlas/pipelines/src/main/java/au/org/ala/distribution/DistributionServiceImpl.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; import java.io.IOException; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -50,12 +52,14 @@ public List getLayers() throws IOException, ExpertDistributio public List findLayersByLsid(String lsid) throws IOException, ExpertDistributionException { + lsid = URLEncoder.encode(lsid, StandardCharsets.UTF_8.toString()); Response> response = service.getLayersByLsid(lsid, "false").execute(); int code = response.code(); if (code >= 200 && code < 300) { List layers = response.body(); return layers; } else { + log.error("Error in finding the expert distribution layer of " + lsid); errorHandler(code, response); return null; } @@ -63,6 +67,7 @@ public List findLayersByLsid(String lsid) public Map outliers(String lsid, Map> points) throws IOException, ExpertDistributionException { + lsid = URLEncoder.encode(lsid, StandardCharsets.UTF_8.toString()); Response> response = service.outliers(lsid, points).execute(); int code = response.code(); if (code >= 200 && code < 300) { diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java index 1204f537be..0e63329748 100644 --- a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java +++ b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/DistributionOutlierTransform.java @@ -65,20 +65,22 @@ public MapElements> toKv() { public Iterable apply( KV> input) { String lsid = input.getKey(); + Iterable records = input.getValue(); Iterator iter = records.iterator(); List outputs = new ArrayList(); - + Map points = new HashMap(); try { DistributionServiceImpl distributionService = DistributionServiceImpl.init(spatialUrl); + List edl = distributionService.findLayersByLsid(lsid); + boolean hasEDL = edl.size() > 0 ? true : false; double distanceToEDL = hasEDL ? 0 : -1; // 0 -inside, -1: no EDL // Available EDLD of this species if (hasEDL) { - Map points = new HashMap(); while (iter.hasNext()) { IndexRecord record = iter.next(); DistributionOutlierRecord dr = convertToDistribution(record, distanceToEDL); @@ -104,14 +106,11 @@ public Iterable apply( } } catch (ExpertDistributionException e) { log.error("Error in processing the species: " + lsid + " . Ignored"); + log.error("Points: " + points); log.error(e.getMessage()); - // throw new RuntimeException( - // "Expert distribution service throws a runtime error, please check - // logs"); } catch (Exception e) { - log.error("Error in processing the species: " + lsid + " . Ignored"); + log.error("Runtime error in processing the species: " + lsid + " . Ignored"); log.error(e.getMessage()); - // throw new RuntimeException("Runtime error: " + e.getMessage()); } return outputs; diff --git a/livingatlas/pipelines/src/main/resources/logback.xml b/livingatlas/pipelines/src/main/resources/logback.xml index 8251880ab9..e0309bd05f 100644 --- a/livingatlas/pipelines/src/main/resources/logback.xml +++ b/livingatlas/pipelines/src/main/resources/logback.xml @@ -3,7 +3,7 @@ true - + diff --git a/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java b/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java index 2ed9594e90..a542131a68 100644 --- a/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java +++ b/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java @@ -4,20 +4,19 @@ import au.org.ala.distribution.DistributionLayer; import au.org.ala.distribution.DistributionServiceImpl; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.junit.Test; public class DistributionOutlierTest { // String spatial_url = "http://devt.ala.org.au:8080/ws/"; String spatial_url = "https://spatial-test.ala.org.au/ws/"; + String local_url = "http://devt.ala.org.au:8080/ws/"; String lsidGreyNurseShark = "urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283"; @Test public void getMultiLayers() { - DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); + DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); try { List layers = impl.findLayersByLsid(lsidGreyNurseShark); assertSame(1, layers.size()); @@ -28,7 +27,7 @@ public void getMultiLayers() { @Test public void outliers() { - DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); + DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); try { Map points = new HashMap(); @@ -76,4 +75,318 @@ public void multiLayers() { e.printStackTrace(); } } + + @Test + public void error_outliers() { + DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); + try { + + Map points = new HashMap(); + // decimalLatitude, decimalLongitude + Map inPoint = new HashMap(); + inPoint.put("decimalLatitude", -32.136495); + inPoint.put("decimalLongitude", 115.746027); + points.put("e512c707-fe92-492c-b869-799c57388c45", inPoint); + + // urn%3Alsid%3Abiodiversity.org.au%3Aafd.taxon%3A34b6d1d7-81be-46cf-b1e1-e80c68d26b34 + // Map results = + // impl.outliers(URLEncoder.encode("urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", StandardCharsets.UTF_8.toString()), points); + Map results = + impl.outliers( + "urn:lsid:biodiversity.org.au:afd.taxon:34b6d1d7-81be-46cf-b1e1-e80c68d26b34", + points); + assertSame(1, results.size()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** return error by chance. Suspect layerStore and postgres issue */ + @Test + public void error_outliers_2() { + DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); + try { + + Map points = new HashMap(); + // decimalLatitude, decimalLongitude + points.put( + "12296131-a31f-4216-85fa-396277d2f4ad", + new HashMap() { + { + put("decimalLatitude", -19.332035); + put("decimalLongitude", 146.755553); + } + }); + points.put( + "373d5681-c70b-4688-a22b-622e64c3eac8", + new HashMap() { + { + put("decimalLatitude", -33.078737); + put("decimalLongitude", 151.608692); + } + }); + points.put( + "4d56b58d-0d29-4829-9208-9d349c609099", + new HashMap() { + { + put("decimalLatitude", -23.35); + put("decimalLongitude", 150.55); + } + }); + points.put( + "4c276c96-3c8d-4069-bf8f-f5d3759b4c6c", + new HashMap() { + { + put("decimalLatitude", -30.383333); + put("decimalLongitude", 153.05); + } + }); + points.put( + "05f4b479-65a1-4bf6-898d-57ec41c0e6f3", + new HashMap() { + { + put("decimalLatitude", -27.548127); + put("decimalLongitude", 153.071967); + } + }); + points.put( + "a3bbedcc-9d22-4430-a9db-510dd27e4678", + new HashMap() { + { + put("decimalLatitude", -30.148182); + put("decimalLongitude", 153.195528); + } + }); + points.put( + "cfc8cc88-15be-4470-985a-3f6c3d329db4", + new HashMap() { + { + put("decimalLatitude", -19.282777); + put("decimalLongitude", 146.800833); + } + }); + points.put( + "583a310b-cbb6-44d4-954d-d4e1eac7fa3b", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "14b24792-bc00-4b96-bad1-76feaf58da94", + new HashMap() { + { + put("decimalLatitude", -17.381555); + put("decimalLongitude", 145.384942); + } + }); + + Map inPoint = new HashMap(); + inPoint.put("decimalLatitude", -23.35); + inPoint.put("decimalLongitude", 150.55); + points.put("8f729f6e-5b74-4a05-a01b-0a81c654aba9", inPoint); + points.put( + "3a2e6757-0dc8-4094-a30e-e7e2ccaeedf9", + new HashMap() { + { + put("decimalLatitude", -27.948758); + put("decimalLongitude", 153.195111); + } + }); + points.put( + "ca985114-3104-4c3a-a706-ccf241a7ff59", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "82840e46-88fa-434a-986f-10091d96ea7c", + new HashMap() { + { + put("decimalLatitude", -23.392031); + put("decimalLongitude", 150.502029); + } + }); + points.put( + "ca985114-3104-4c3a-a706-ccf241a7ff59", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "7d082056-4712-4d71-8b4c-b779e3910c87", + new HashMap() { + { + put("decimalLatitude", -27.941442); + put("decimalLongitude", 153.199367); + } + }); + + // urn%3Alsid%3Abiodiversity.org.au%3Aafd.taxon%3A34b6d1d7-81be-46cf-b1e1-e80c68d26b34 + + Map results = + impl.outliers( + "urn:lsid:biodiversity.org.au:afd.taxon:21686993-4d8f-4ae5-8f81-017c8528c6d5", + points); + + // Test grey shark on local + // Map results = impl.outliers(" + // urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", points); + + assertSame(0, results.size()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** return error by chance. Suspect layerStore and postgres issue */ + @Test + public void error_test_local() { + DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); + try { + + Map points = new HashMap(); + // decimalLatitude, decimalLongitude + points.put( + "12296131-a31f-4216-85fa-396277d2f4ad", + new HashMap() { + { + put("decimalLatitude", -17.54858); + put("decimalLongitude", 131.471238); + } + }); + points.put( + "373d5681-c70b-4688-a22b-622e64c3eac8", + new HashMap() { + { + put("decimalLatitude", -33.078737); + put("decimalLongitude", 151.608692); + } + }); + points.put( + "4d56b58d-0d29-4829-9208-9d349c609099", + new HashMap() { + { + put("decimalLatitude", -23.35); + put("decimalLongitude", 150.55); + } + }); + points.put( + "4c276c96-3c8d-4069-bf8f-f5d3759b4c6c", + new HashMap() { + { + put("decimalLatitude", -30.383333); + put("decimalLongitude", 153.05); + } + }); + points.put( + "05f4b479-65a1-4bf6-898d-57ec41c0e6f3", + new HashMap() { + { + put("decimalLatitude", -27.548127); + put("decimalLongitude", 153.071967); + } + }); + points.put( + "a3bbedcc-9d22-4430-a9db-510dd27e4678", + new HashMap() { + { + put("decimalLatitude", -30.148182); + put("decimalLongitude", 153.195528); + } + }); + points.put( + "cfc8cc88-15be-4470-985a-3f6c3d329db4", + new HashMap() { + { + put("decimalLatitude", -19.282777); + put("decimalLongitude", 146.800833); + } + }); + points.put( + "583a310b-cbb6-44d4-954d-d4e1eac7fa3b", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "14b24792-bc00-4b96-bad1-76feaf58da94", + new HashMap() { + { + put("decimalLatitude", -17.381555); + put("decimalLongitude", 145.384942); + } + }); + + points.put( + "ca985114-3104-4c3a-a706-ccf241a7ff59", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "82840e46-88fa-434a-986f-10091d96ea7c", + new HashMap() { + { + put("decimalLatitude", -23.392031); + put("decimalLongitude", 150.502029); + } + }); + points.put( + "ca985114-3104-4c3a-a706-ccf241a7ff59", + new HashMap() { + { + put("decimalLatitude", -28.840865); + put("decimalLongitude", 153.449584); + } + }); + points.put( + "7d082056-4712-4d71-8b4c-b779e3910c87", + new HashMap() { + { + put("decimalLatitude", -27.941442); + put("decimalLongitude", 153.199367); + } + }); + + // Test grey shark on local + Map results = + impl.outliers( + "urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", + points); + + assertSame(9, results.size()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // [SPATIAL-SERVICE] 13:26:51 a.o.a.l.DistributionController : Runtime error in calculating + // outliers of lsid: urn:lsid:biodiversity.org.au:afd.taxon:21686993-4d8f-4ae5-8f81-017c8528c6d5 + // [8f729f6e-5b74-4a05-a01b-0a81c654aba9:[decimalLongitude:150.55, decimalLatitude:-23.35], + // 3a2e6757-0dc8-4094-a30e-e7e2ccaeedf9:[decimalLongitude:153.195111, decimalLatitude:-27.948758], + // ca985114-3104-4c3a-a706-ccf241a7ff59:[decimalLongitude:153.449584, decimalLatitude:-28.840865], + // 82840e46-88fa-434a-986f-10091d96ea7c:[decimalLongitude:150.502029, decimalLatitude:-23.392031], + // 7d082056-4712-4d71-8b4c-b779e3910c87:[decimalLongitude:153.199367, decimalLatitude:-27.941442], + // 14b24792-bc00-4b96-bad1-76feaf58da94:[decimalLongitude:145.384942, decimalLatitude:-17.381555], + // 583a310b-cbb6-44d4-954d-d4e1eac7fa3b:[decimalLongitude:153.449584, decimalLatitude:-28.840865], + // cfc8cc88-15be-4470-985a-3f6c3d329db4:[decimalLongitude:146.800833, decimalLatitude:-19.282777], + // a3bbedcc-9d22-4430-a9db-510dd27e4678:[decimalLongitude:153.195528, decimalLatitude:-30.148182], + // 05f4b479-65a1-4bf6-898d-57ec41c0e6f3:[decimalLongitude:153.071967, decimalLatitude:-27.548127], + // 4c276c96-3c8d-4069-bf8f-f5d3759b4c6c:[decimalLongitude:153.05, decimalLatitude:-30.383333], + // 4d56b58d-0d29-4829-9208-9d349c609099:[decimalLongitude:150.55, decimalLatitude:-23.35], + // 373d5681-c70b-4688-a22b-622e64c3eac8:[decimalLongitude:151.608692, decimalLatitude:-33.078737], + // 12296131-a31f-4216-85fa-396277d2f4ad:[decimalLongitude:146.755553, decimalLatitude:-19.332035]] + // [SPATIAL-SERVICE] 13:26:51 a.o.a.l.DistributionController : java.lang.NumberFormatException: + // For input string: ""9"" + }