Fixes #190

SIDN · Sep 29, 2022 · 516619f · 516619f
1 parent fecbc71
commit 516619f
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 4 deletions.
diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
 
 	<groupId>nl.sidnlabs</groupId>
 	<artifactId>entrada</artifactId>
-	<version>2.4.6</version>
+	<version>2.4.7</version>
 	<name>${project.groupId}:${project.artifactId}</name>
 	<description>Convert en enrich DNS data to Parquet format</description>
 	<url>https://github.com/SIDN/entrada</url>

diff --git a/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java b/src/main/java/nl/sidnlabs/entrada/engine/ImpalaQueryEngine.java
@@ -50,7 +50,8 @@ public boolean postCompact(TablePartition p) {
     log.info("Perform post-compaction actions");
 
     log
-        .info("Perform post-compaction actions, refresh and compute stats for table: {}",
+        .info(
+            "Perform post-compaction actions, invalidate metadata and compute stats for table: {}",
             p.getTable());
     Map<String, Object> values =
         templateValues(p.getTable(), p.getYear(), p.getMonth(), p.getDay(), p.getServer());
@@ -78,10 +79,13 @@ public boolean postPurge(TablePartition p) {
   }
 
   private boolean invalidate(String table, Map<String, Object> values) {
-    String sqlRefresh = TemplateUtil
+
+    log.info("Invalidate metadata, table: {}", table);
+
+    String sql = TemplateUtil
         .template(new ClassPathResource("/sql/impala/invalidate-metadata.sql", getClass()), values);
 
-    return execute(sqlRefresh);
+    return execute(sql);
   }
 
 

diff --git a/src/main/java/nl/sidnlabs/entrada/service/UploadService.java b/src/main/java/nl/sidnlabs/entrada/service/UploadService.java
@@ -156,6 +156,31 @@ public void upload(Map<String, Set<Partition>> partitions, boolean clean) {
           }
 
           if (filesToUpload.size() > 0) {
+
+            // make sure correct permissions are set, do not set recursive at year/month level
+            // when many files this will take long time
+            fmOutput.chown(FileUtil.appendPath(dstLocation, partition.toYear()), false);
+            fmOutput.chmod(FileUtil.appendPath(dstLocation, partition.toYear()), false);
+
+            fmOutput
+                .chown(FileUtil.appendPath(dstLocation, partition.toYear(), partition.toMonth()),
+                    false);
+            fmOutput
+                .chmod(FileUtil.appendPath(dstLocation, partition.toYear(), partition.toMonth()),
+                    false);
+
+            // recursive update under the day level
+            fmOutput
+                .chown(FileUtil
+                    .appendPath(dstLocation, partition.toYear(), partition.toMonth(),
+                        partition.toDay()),
+                    true);
+            fmOutput
+                .chmod(FileUtil
+                    .appendPath(dstLocation, partition.toYear(), partition.toMonth(),
+                        partition.toDay()),
+                    true);
+
             queryEngine.addPartition("icmp", tableNameIcmp, partition);
           }
         }