From 516619f5a04d30403714c5ecf7715f2bf97c718a Mon Sep 17 00:00:00 2001
From: Maarten <maarten.wullink@sidn.nl>
Date: Thu, 29 Sep 2022 08:31:53 +0200
Subject: [PATCH] Fixes #190

---
 pom.xml                                       |  2 +-
 .../entrada/engine/ImpalaQueryEngine.java     | 10 +++++---
 .../entrada/service/UploadService.java        | 25 +++++++++++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index f1bb209..c57ad51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
 
 	<groupId>nl.sidnlabs</groupId>
 	<artifactId>entrada</artifactId>
-	<version>2.4.6</version>
+	<version>2.4.7</version>
 	<name>${project.groupId}:${project.artifactId}</name>
 	<description>Convert en enrich DNS data to Parquet format</description>
 	<url>https://github.com/SIDN/entrada</url>
diff --git a/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java b/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java
index b44a644..bd5b1d3 100644
--- a/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java
+++ b/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java
@@ -50,7 +50,8 @@ public boolean postCompact(TablePartition p) {
     log.info("Perform post-compaction actions");
 
     log
-        .info("Perform post-compaction actions, refresh and compute stats for table: {}",
+        .info(
+            "Perform post-compaction actions, invalidate metadata and compute stats for table: {}",
             p.getTable());
     Map<String, Object> values =
         templateValues(p.getTable(), p.getYear(), p.getMonth(), p.getDay(), p.getServer());
@@ -78,10 +79,13 @@ public boolean postPurge(TablePartition p) {
   }
 
   private boolean invalidate(String table, Map<String, Object> values) {
-    String sqlRefresh = TemplateUtil
+
+    log.info("Invalidate metadata, table: {}", table);
+
+    String sql = TemplateUtil
         .template(new ClassPathResource("/sql/impala/invalidate-metadata.sql", getClass()), values);
 
-    return execute(sqlRefresh);
+    return execute(sql);
   }
 
 
diff --git a/src/main/java/nl/sidnlabs/entrada/service/UploadService.java b/src/main/java/nl/sidnlabs/entrada/service/UploadService.java
index 7756314..1b09b7d 100644
--- a/src/main/java/nl/sidnlabs/entrada/service/UploadService.java
+++ b/src/main/java/nl/sidnlabs/entrada/service/UploadService.java
@@ -156,6 +156,31 @@ public void upload(Map<String, Set<Partition>> partitions, boolean clean) {
           }
 
           if (filesToUpload.size() > 0) {
+
+            // make sure correct permissions are set, do not set recursive at year/month level
+            // when many files this will take long time
+            fmOutput.chown(FileUtil.appendPath(dstLocation, partition.toYear()), false);
+            fmOutput.chmod(FileUtil.appendPath(dstLocation, partition.toYear()), false);
+
+            fmOutput
+                .chown(FileUtil.appendPath(dstLocation, partition.toYear(), partition.toMonth()),
+                    false);
+            fmOutput
+                .chmod(FileUtil.appendPath(dstLocation, partition.toYear(), partition.toMonth()),
+                    false);
+
+            // recursive update under the day level
+            fmOutput
+                .chown(FileUtil
+                    .appendPath(dstLocation, partition.toYear(), partition.toMonth(),
+                        partition.toDay()),
+                    true);
+            fmOutput
+                .chmod(FileUtil
+                    .appendPath(dstLocation, partition.toYear(), partition.toMonth(),
+                        partition.toDay()),
+                    true);
+
             queryEngine.addPartition("icmp", tableNameIcmp, partition);
           }
         }