#622 impl(Only calculated new added records)

gbif · Mar 3, 2022 · 15b277d · 15b277d
1 parent a295d7d
commit 15b277d
Showing 1 changed file with 33 additions and 5 deletions.
diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/utils/ALAFsUtils.java b/livingatlas/pipelines/src/main/java/au/org/ala/utils/ALAFsUtils.java
@@ -72,10 +72,40 @@ public static String buildPathSamplingUsingTargetPath(AllDatasetsPipelinesOption
   }
 
   /**
-   * Build a path to outlier records. {fsPath}/pipelines-outlier/{datasetId}
+   * NOTE: It will delete the existing folder Build a path to outlier records.
+   * {fsPath}/pipelines-outlier/{datasetId} {fsPath}/pipelines-outlier/all
+   */
+  public static String buildPathOutlierUsingTargetPath(
+      AllDatasetsPipelinesOptions options, boolean delete) throws IOException {
+    // default: {fsPath}/pipelines-outlier
+    FileSystem fs =
+        FileSystemFactory.getInstance(options.getHdfsSiteConfig(), options.getCoreSiteConfig())
+            .getFs(options.getTargetPath());
+
+    String outputPath = PathBuilder.buildPath(options.getTargetPath()).toString();
+
+    // {fsPath}/pipelines-outlier/{datasetId}
+    if (options.getDatasetId() != null && !"all".equalsIgnoreCase(options.getDatasetId())) {
+      outputPath = PathBuilder.buildPath(outputPath, options.getDatasetId()).toString();
+    } else {
+      // {fsPath}/pipelines-outlier/all
+      outputPath = PathBuilder.buildPath(outputPath, "all").toString();
+    }
+    // delete previous runs
+    if (delete)
+      FsUtils.deleteIfExist(options.getHdfsSiteConfig(), options.getCoreSiteConfig(), outputPath);
+    else {
+      if (!exists(fs, outputPath)) ALAFsUtils.createDirectory(fs, outputPath);
+    }
+
+    return outputPath;
+  }
+
+  /**
+   * Get an output path to outlier records. {fsPath}/pipelines-outlier/{datasetId}
    * {fsPath}/pipelines-outlier/all
    */
-  public static String buildPathOutlierUsingTargetPath(AllDatasetsPipelinesOptions options)
+  public static String getOutlierTargetPath(AllDatasetsPipelinesOptions options)
       throws IOException {
     // default: {fsPath}/pipelines-outlier
     FileSystem fs =
@@ -91,9 +121,7 @@ public static String buildPathOutlierUsingTargetPath(AllDatasetsPipelinesOptions
       // {fsPath}/pipelines-outlier/all
       outputPath = PathBuilder.buildPath(outputPath, "all").toString();
     }
-    // delete previous runs
-    FsUtils.deleteIfExist(options.getHdfsSiteConfig(), options.getCoreSiteConfig(), outputPath);
-    ALAFsUtils.createDirectory(fs, outputPath);
+
     return outputPath;
   }