From 6bf5a9467a05b4e2ee405c9e2e1456f991f73385 Mon Sep 17 00:00:00 2001 From: Qifeng Date: Fri, 14 Jan 2022 11:24:45 +1100 Subject: [PATCH] #622 fix(remove log4j config from script) --- .../pipelines/src/main/resources/logback.xml | 11 - .../ala/outlier/DistributionOutlierTest.java | 309 +----------------- livingatlas/scripts/la-pipelines | 18 +- 3 files changed, 18 insertions(+), 320 deletions(-) diff --git a/livingatlas/pipelines/src/main/resources/logback.xml b/livingatlas/pipelines/src/main/resources/logback.xml index e0309bd05f..40f1c31ad1 100644 --- a/livingatlas/pipelines/src/main/resources/logback.xml +++ b/livingatlas/pipelines/src/main/resources/logback.xml @@ -51,15 +51,4 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java b/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java index a542131a68..f51edcdbd2 100644 --- a/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java +++ b/livingatlas/pipelines/src/test/java/au/org/ala/outlier/DistributionOutlierTest.java @@ -10,16 +10,15 @@ public class DistributionOutlierTest { // String spatial_url = "http://devt.ala.org.au:8080/ws/"; String spatial_url = "https://spatial-test.ala.org.au/ws/"; - String local_url = "http://devt.ala.org.au:8080/ws/"; String lsidGreyNurseShark = "urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283"; @Test public void getMultiLayers() { - DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); + DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); try { List layers = impl.findLayersByLsid(lsidGreyNurseShark); - assertSame(1, layers.size()); + assertSame(5, layers.size()); } catch (Exception e) { System.out.println(e.getMessage()); } @@ -27,7 +26,7 @@ public void getMultiLayers() { @Test public void outliers() { - DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); + DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); try { Map points = new HashMap(); @@ -77,316 +76,24 @@ public void multiLayers() { } @Test - public void error_outliers() { + public void url_as_id_outliers() { DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); try { Map points = new HashMap(); // decimalLatitude, decimalLongitude Map inPoint = new HashMap(); - inPoint.put("decimalLatitude", -32.136495); - inPoint.put("decimalLongitude", 115.746027); + inPoint.put("decimalLatitude", -37.151663); + inPoint.put("decimalLongitude", 142.854663); points.put("e512c707-fe92-492c-b869-799c57388c45", inPoint); - // urn%3Alsid%3Abiodiversity.org.au%3Aafd.taxon%3A34b6d1d7-81be-46cf-b1e1-e80c68d26b34 - // Map results = - // impl.outliers(URLEncoder.encode("urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", StandardCharsets.UTF_8.toString()), points); - Map results = - impl.outliers( - "urn:lsid:biodiversity.org.au:afd.taxon:34b6d1d7-81be-46cf-b1e1-e80c68d26b34", - points); - assertSame(1, results.size()); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** return error by chance. Suspect layerStore and postgres issue */ - @Test - public void error_outliers_2() { - DistributionServiceImpl impl = DistributionServiceImpl.init(spatial_url); - try { - - Map points = new HashMap(); - // decimalLatitude, decimalLongitude - points.put( - "12296131-a31f-4216-85fa-396277d2f4ad", - new HashMap() { - { - put("decimalLatitude", -19.332035); - put("decimalLongitude", 146.755553); - } - }); - points.put( - "373d5681-c70b-4688-a22b-622e64c3eac8", - new HashMap() { - { - put("decimalLatitude", -33.078737); - put("decimalLongitude", 151.608692); - } - }); - points.put( - "4d56b58d-0d29-4829-9208-9d349c609099", - new HashMap() { - { - put("decimalLatitude", -23.35); - put("decimalLongitude", 150.55); - } - }); - points.put( - "4c276c96-3c8d-4069-bf8f-f5d3759b4c6c", - new HashMap() { - { - put("decimalLatitude", -30.383333); - put("decimalLongitude", 153.05); - } - }); - points.put( - "05f4b479-65a1-4bf6-898d-57ec41c0e6f3", - new HashMap() { - { - put("decimalLatitude", -27.548127); - put("decimalLongitude", 153.071967); - } - }); - points.put( - "a3bbedcc-9d22-4430-a9db-510dd27e4678", - new HashMap() { - { - put("decimalLatitude", -30.148182); - put("decimalLongitude", 153.195528); - } - }); - points.put( - "cfc8cc88-15be-4470-985a-3f6c3d329db4", - new HashMap() { - { - put("decimalLatitude", -19.282777); - put("decimalLongitude", 146.800833); - } - }); - points.put( - "583a310b-cbb6-44d4-954d-d4e1eac7fa3b", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "14b24792-bc00-4b96-bad1-76feaf58da94", - new HashMap() { - { - put("decimalLatitude", -17.381555); - put("decimalLongitude", 145.384942); - } - }); - - Map inPoint = new HashMap(); - inPoint.put("decimalLatitude", -23.35); - inPoint.put("decimalLongitude", 150.55); - points.put("8f729f6e-5b74-4a05-a01b-0a81c654aba9", inPoint); - points.put( - "3a2e6757-0dc8-4094-a30e-e7e2ccaeedf9", - new HashMap() { - { - put("decimalLatitude", -27.948758); - put("decimalLongitude", 153.195111); - } - }); - points.put( - "ca985114-3104-4c3a-a706-ccf241a7ff59", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "82840e46-88fa-434a-986f-10091d96ea7c", - new HashMap() { - { - put("decimalLatitude", -23.392031); - put("decimalLongitude", 150.502029); - } - }); - points.put( - "ca985114-3104-4c3a-a706-ccf241a7ff59", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "7d082056-4712-4d71-8b4c-b779e3910c87", - new HashMap() { - { - put("decimalLatitude", -27.941442); - put("decimalLongitude", 153.199367); - } - }); - - // urn%3Alsid%3Abiodiversity.org.au%3Aafd.taxon%3A34b6d1d7-81be-46cf-b1e1-e80c68d26b34 - - Map results = - impl.outliers( - "urn:lsid:biodiversity.org.au:afd.taxon:21686993-4d8f-4ae5-8f81-017c8528c6d5", - points); - - // Test grey shark on local - // Map results = impl.outliers(" - // urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", points); - + //Map results = impl.outliers(URLEncoder.encode("urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", StandardCharsets.UTF_8.toString()), points); + Map results = impl.outliers("https://id.biodiversity.org.au/node/apni/2908371", points); assertSame(0, results.size()); } catch (Exception e) { e.printStackTrace(); } } - /** return error by chance. Suspect layerStore and postgres issue */ - @Test - public void error_test_local() { - DistributionServiceImpl impl = DistributionServiceImpl.init(local_url); - try { - - Map points = new HashMap(); - // decimalLatitude, decimalLongitude - points.put( - "12296131-a31f-4216-85fa-396277d2f4ad", - new HashMap() { - { - put("decimalLatitude", -17.54858); - put("decimalLongitude", 131.471238); - } - }); - points.put( - "373d5681-c70b-4688-a22b-622e64c3eac8", - new HashMap() { - { - put("decimalLatitude", -33.078737); - put("decimalLongitude", 151.608692); - } - }); - points.put( - "4d56b58d-0d29-4829-9208-9d349c609099", - new HashMap() { - { - put("decimalLatitude", -23.35); - put("decimalLongitude", 150.55); - } - }); - points.put( - "4c276c96-3c8d-4069-bf8f-f5d3759b4c6c", - new HashMap() { - { - put("decimalLatitude", -30.383333); - put("decimalLongitude", 153.05); - } - }); - points.put( - "05f4b479-65a1-4bf6-898d-57ec41c0e6f3", - new HashMap() { - { - put("decimalLatitude", -27.548127); - put("decimalLongitude", 153.071967); - } - }); - points.put( - "a3bbedcc-9d22-4430-a9db-510dd27e4678", - new HashMap() { - { - put("decimalLatitude", -30.148182); - put("decimalLongitude", 153.195528); - } - }); - points.put( - "cfc8cc88-15be-4470-985a-3f6c3d329db4", - new HashMap() { - { - put("decimalLatitude", -19.282777); - put("decimalLongitude", 146.800833); - } - }); - points.put( - "583a310b-cbb6-44d4-954d-d4e1eac7fa3b", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "14b24792-bc00-4b96-bad1-76feaf58da94", - new HashMap() { - { - put("decimalLatitude", -17.381555); - put("decimalLongitude", 145.384942); - } - }); - - points.put( - "ca985114-3104-4c3a-a706-ccf241a7ff59", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "82840e46-88fa-434a-986f-10091d96ea7c", - new HashMap() { - { - put("decimalLatitude", -23.392031); - put("decimalLongitude", 150.502029); - } - }); - points.put( - "ca985114-3104-4c3a-a706-ccf241a7ff59", - new HashMap() { - { - put("decimalLatitude", -28.840865); - put("decimalLongitude", 153.449584); - } - }); - points.put( - "7d082056-4712-4d71-8b4c-b779e3910c87", - new HashMap() { - { - put("decimalLatitude", -27.941442); - put("decimalLongitude", 153.199367); - } - }); - - // Test grey shark on local - Map results = - impl.outliers( - "urn:lsid:biodiversity.org.au:afd.taxon:0c3e2403-05c4-4a43-8019-30e6d657a283", - points); - - assertSame(9, results.size()); - } catch (Exception e) { - e.printStackTrace(); - } - } - - // [SPATIAL-SERVICE] 13:26:51 a.o.a.l.DistributionController : Runtime error in calculating - // outliers of lsid: urn:lsid:biodiversity.org.au:afd.taxon:21686993-4d8f-4ae5-8f81-017c8528c6d5 - // [8f729f6e-5b74-4a05-a01b-0a81c654aba9:[decimalLongitude:150.55, decimalLatitude:-23.35], - // 3a2e6757-0dc8-4094-a30e-e7e2ccaeedf9:[decimalLongitude:153.195111, decimalLatitude:-27.948758], - // ca985114-3104-4c3a-a706-ccf241a7ff59:[decimalLongitude:153.449584, decimalLatitude:-28.840865], - // 82840e46-88fa-434a-986f-10091d96ea7c:[decimalLongitude:150.502029, decimalLatitude:-23.392031], - // 7d082056-4712-4d71-8b4c-b779e3910c87:[decimalLongitude:153.199367, decimalLatitude:-27.941442], - // 14b24792-bc00-4b96-bad1-76feaf58da94:[decimalLongitude:145.384942, decimalLatitude:-17.381555], - // 583a310b-cbb6-44d4-954d-d4e1eac7fa3b:[decimalLongitude:153.449584, decimalLatitude:-28.840865], - // cfc8cc88-15be-4470-985a-3f6c3d329db4:[decimalLongitude:146.800833, decimalLatitude:-19.282777], - // a3bbedcc-9d22-4430-a9db-510dd27e4678:[decimalLongitude:153.195528, decimalLatitude:-30.148182], - // 05f4b479-65a1-4bf6-898d-57ec41c0e6f3:[decimalLongitude:153.071967, decimalLatitude:-27.548127], - // 4c276c96-3c8d-4069-bf8f-f5d3759b4c6c:[decimalLongitude:153.05, decimalLatitude:-30.383333], - // 4d56b58d-0d29-4829-9208-9d349c609099:[decimalLongitude:150.55, decimalLatitude:-23.35], - // 373d5681-c70b-4688-a22b-622e64c3eac8:[decimalLongitude:151.608692, decimalLatitude:-33.078737], - // 12296131-a31f-4216-85fa-396277d2f4ad:[decimalLongitude:146.755553, decimalLatitude:-19.332035]] - // [SPATIAL-SERVICE] 13:26:51 a.o.a.l.DistributionController : java.lang.NumberFormatException: - // For input string: ""9"" } diff --git a/livingatlas/scripts/la-pipelines b/livingatlas/scripts/la-pipelines index fffc864e21..42d79b76af 100755 --- a/livingatlas/scripts/la-pipelines +++ b/livingatlas/scripts/la-pipelines @@ -174,10 +174,13 @@ if ($dry_run); then _D=echo; else _D=; fi # Set default config locations for Production and Development if [[ -z $config ]]; then config=$CONFIG_DIR/la-pipelines.yaml,$CONFIG_DIR/la-pipelines-local.yaml; fi -if [[ $PROD = false && $no_colors = false ]]; then logConfig=$PWD/../pipelines/src/main/resources/log4j-colorized.properties; fi -if [[ $PROD = false && $no_colors = true ]]; then logConfig=$PWD/../pipelines/src/main/resources/log4j.properties; fi -if [[ $PROD = true && $no_colors = false ]]; then logConfig=$CONFIG_DIR/log4j-colorized.properties; fi -if [[ $PROD = true && $no_colors = true ]]; then logConfig=$CONFIG_DIR/log4j.properties; fi +#if [[ $PROD = false && $no_colors = false ]]; then logConfig=$PWD/../pipelines/src/main/resources/log4j-colorized.properties; fi +#if [[ $PROD = false && $no_colors = true ]]; then logConfig=$PWD/../pipelines/src/main/resources/log4j.properties; fi +#if [[ $PROD = true && $no_colors = false ]]; then logConfig=$CONFIG_DIR/log4j-colorized.properties; fi +#if [[ $PROD = true && $no_colors = true ]]; then logConfig=$CONFIG_DIR/log4j.properties; fi + +if [[ $PROD = false ]] ; then logConfig=$PWD/../pipelines/src/main/resources/logback.xml ; fi +if [[ $PROD = true ]] ; then logConfig=$CONFIG_DIR/logback.xml ; fi # Let's parse dr variable in order to allow multiple drs drs=("${dr[@]}") @@ -202,7 +205,7 @@ fi log.info Config: $config log.info Extra arguments: $ARGS -log.debug Logs without colors: $no_colors +#log.debug Logs without colors: $no_colors log.debug log4j config: $logConfig # Convert config to a list of files space separated @@ -297,8 +300,8 @@ ARGS="$ARGS $FS_PATH_ARG" # if ( $debug) ; then EXTRA_JVM_CLI_ARGS="-Dlog4j.debug"; fi # Set log4j configuration for v1 and v2 -EXTRA_JVM_CLI_ARGS="$EXTRA_JVM_CLI_ARGS -Dlog4j.configuration=file://$logConfig -Dlog4j.configurationFile=file://$logConfig" -log.debug EXTRA_JVM_CLI_ARGS: $EXTRA_JVM_CLI_ARGS +#EXTRA_JVM_CLI_ARGS="$EXTRA_JVM_CLI_ARGS -Dlog4j.configuration=file://$logConfig -Dlog4j.configurationFile=file://$logConfig" +#log.debug EXTRA_JVM_CLI_ARGS: $EXTRA_JVM_CLI_ARGS function logStepStart() { local name="$1" dr="$2" @@ -428,7 +431,6 @@ function spark-cluster-pipeline() { $(toArg $SH_ARGS driver-memory) \ --class $CLASS \ --master $SPARK_MASTER \ - --driver-java-options "-Dlog4j.configuration=file:$CONFIG_DIR/log4j.properties" \ $PIPELINES_JAR \ --datasetId=$dr \ --config=$config $ARGS \