Caching improvements: to avoid repeated feature extraction caching is…

… now implemented with a content based hash and not on a path of a file.
JorenSix · Oct 11, 2022 · c2cd1cf · c2cd1cf
1 parent 70a3168
commit c2cd1cf
Show file tree

Hide file tree

Showing 22 changed files with 1,021 additions and 372 deletions.
diff --git a/resources/defaults/logging.properties b/resources/defaults/logging.properties
@@ -55,4 +55,5 @@ java.util.logging.FileHandler.limit=5000000
 # integer to the base file name:
 java.util.logging.FileHandler.count=5
 
-# Style of output (Simple or XML):
+# Style of output (Simple or XML):
+java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
diff --git a/src/main/java/be/panako/cli/Application.java b/src/main/java/be/panako/cli/Application.java
@@ -140,9 +140,9 @@ protected boolean checkFile(String file){
 		File f = new File(file);
 		boolean fileOk = false;
 		if(f.exists() && f.canRead()){
-			fileOk = true;
+			fileOk = FileUtils.checkFileSize(f,Config.getInt(Key.MAX_FILE_SIZE));
 		}else{
-			String message = "Could not read " + f.getAbsolutePath() + " it does not exist or is not accesible at the moment.)";
+			String message = "Could not read " + f.getAbsolutePath() + " it does not exist or is not accessible at the moment.)";
 			LOG.warning(message);
 			System.out.println(message);
 		}

diff --git a/src/main/java/be/panako/cli/Delete.java b/src/main/java/be/panako/cli/Delete.java
@@ -45,10 +45,7 @@
 
 import be.panako.strategy.Strategy;
 import be.panako.strategy.olaf.OlafStrategy;
-import be.panako.util.Config;
-import be.panako.util.Key;
-import be.panako.util.StopWatch;
-import be.panako.util.TimeUnit;
+import be.panako.util.*;
 
 /**
  * Delete fingerptings from the index.
@@ -121,42 +118,29 @@ public DeleteTask(File file,int taskID,int totalTasks){
 		public void run() {
 
 			StopWatch w = new StopWatch();
-			if(checkFile(file)){
-
-				Strategy strategy =  Strategy.getInstance();
-
-				boolean hasResource = false;
-				hasResource =  strategy.hasResource(file.getAbsolutePath());
-
-				String message=null;
-				if(hasResource){
-					message = String.format("%d/%d;%s;%s;%s",taskID,totalTasks,file.getName(),StopWatch.toTime("", 0),"Deletion skipped: resource not in the key value store;");
-				}else{
-					double durationInSeconds = strategy.delete(file.getAbsolutePath());
-
-					double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
-					String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
-					String cpuTimeDuration = w.formattedToString();
-					double timeRatio = durationInSeconds/cpuSecondsPassed;
-					message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);			
-				}
-				LOG.info(message);
-				System.out.println(message);
-			}
-		}
-
-		private boolean checkFile(File file){
-			boolean fileOk = false;
-			//file must be smaller than a configured number of bytes
-			if(file.length() != 0 && file.length() < Config.getInt(Key.MAX_FILE_SIZE)){
-				fileOk = true;
+			Strategy strategy =  Strategy.getInstance();
+
+			boolean hasResource = false;
+			hasResource =  strategy.hasResource(file.getAbsolutePath());
+
+			String message=null;
+			if(hasResource){
+				message = String.format("%d/%d;%s;%s;%s",taskID,totalTasks,file.getName(),StopWatch.toTime("", 0),"Deletion skipped: resource not in the key value store;");
 			}else{
-				String message = "Could not process " + file.getName() + " it has an unacceptable file size: zero or larger than " + Config.getInt(Key.MAX_FILE_SIZE) + "bytes ).";
-				LOG.warning(message);
-				System.out.println(message);
+				double durationInSeconds = strategy.delete(file.getAbsolutePath());
+
+				double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
+				String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
+				String cpuTimeDuration = w.formattedToString();
+				double timeRatio = durationInSeconds/cpuSecondsPassed;
+				message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
 			}
-			return fileOk;
+			LOG.info(message);
+			System.out.println(message);
+
 		}
+
+
 	}
 
 

diff --git a/src/main/java/be/panako/cli/Monitor.java b/src/main/java/be/panako/cli/Monitor.java
@@ -83,14 +83,14 @@ public void run(String... args) {
 		if(hasArgument("debug", args) || processors==1){
 			int taskNumber = 1;
 			for(File file: files){
-				new Monitor.MonitorTask(file.getPath(),taskNumber,files.size()).run();
+				new Monitor.MonitorTask(file.getAbsolutePath(),taskNumber,files.size()).run();
 				taskNumber++;
 			}
 		}else{
 			ExecutorService executor = Executors.newFixedThreadPool(processors);
 			int taskNumber = 1;
 			for(File file: files){
-				executor.submit(new Monitor.MonitorTask(file.getPath(),taskNumber,files.size()));
+				executor.submit(new Monitor.MonitorTask(file.getAbsolutePath(),taskNumber,files.size()));
 				taskNumber++;
 			}
 			executor.shutdown();

diff --git a/src/main/java/be/panako/cli/Print.java b/src/main/java/be/panako/cli/Print.java
@@ -59,7 +59,7 @@ public void run(String... args) {
 		Strategy strategy = Strategy.getInstance();
 
 		for(File file: files){
-			strategy.print(file.getPath(),sonicVisualizerOutput);
+			strategy.print(file.getAbsolutePath(),sonicVisualizerOutput);
 		}
 
 	}

diff --git a/src/main/java/be/panako/cli/Query.java b/src/main/java/be/panako/cli/Query.java
@@ -70,14 +70,14 @@ public void run(String... args) {
 		if(hasArgument("debug", args) || processors==1){
 			int taskNumber = 1;
 			for(File file: files){
-				new QueryTask(file.getPath(),taskNumber,files.size()).run();
+				new QueryTask(file.getAbsolutePath(),taskNumber,files.size()).run();
 				taskNumber++;
 			}
 		}else{
 			ExecutorService executor = Executors.newFixedThreadPool(processors);
 			int taskNumber = 1;
 			for(File file: files){
-				executor.submit(new QueryTask(file.getPath(),taskNumber,files.size()));
+				executor.submit(new QueryTask(file.getAbsolutePath(),taskNumber,files.size()));
 				taskNumber++;
 			}
 			executor.shutdown();

diff --git a/src/main/java/be/panako/cli/Resolve.java b/src/main/java/be/panako/cli/Resolve.java
@@ -47,11 +47,11 @@ class Resolve extends Application {
 
 	@Override
 	public void run(String... args) {
-		Strategy strat = Strategy.getInstance();
+		Strategy strategy = Strategy.getInstance();
 		List<File> files = getFilesFromArguments(args);
 
 		for(File f : files) {
-			System.out.println(strat.resolve(f.getPath()));
+			System.out.println(strategy.resolve(f.getAbsolutePath()));
 		}
 	}
 

diff --git a/src/main/java/be/panako/cli/Store.java b/src/main/java/be/panako/cli/Store.java
@@ -42,10 +42,7 @@
 import java.util.logging.Logger;
 
 import be.panako.strategy.Strategy;
-import be.panako.util.Config;
-import be.panako.util.Key;
-import be.panako.util.StopWatch;
-import be.panako.util.TimeUnit;
+import be.panako.util.*;
 
 /**
  * Store audio fingerptings in the storage. 
@@ -116,49 +113,31 @@ public StoreTask(File file,int taskID,int totalTasks){
 
 		@Override
 		public void run() {
-
 			StopWatch w = new StopWatch();
-			if(checkFile(file)){
-
-				Strategy strategy = Strategy.getInstance();
-
-				boolean isDouble = false;				
-				if(Config.getBoolean(Key.CHECK_DUPLICATE_FILE_NAMES) ){
-					isDouble =  strategy.hasResource(file.getAbsolutePath());
-				}
+			Strategy strategy = Strategy.getInstance();
 
-				String message=null;
-				if(isDouble){
-					message = String.format("%d/%d;%s;%s",taskID,totalTasks,file.getName(),"Skipped: resource already stored;");
-				}else{
-					double durationInSeconds = strategy.store(file.getAbsolutePath(), file.getName());
-					double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
-					String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
-					String cpuTimeDuration = w.formattedToString();
-					double timeRatio = durationInSeconds/cpuSecondsPassed;
-					message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);			
-				}
-				LOG.info(message);
-				System.out.println(message);
+			boolean isDouble = false;
+			if(Config.getBoolean(Key.CHECK_DUPLICATE_FILE_NAMES) ){
+				isDouble =  strategy.hasResource(file.getAbsolutePath());
 			}
-		}
-
-		private boolean checkFile(File file){
-			boolean fileOk = false;
-
-			//file must be smaller than a configured number of bytes
-			long maxFileSize =  Config.getInt(Key.MAX_FILE_SIZE);
-			//from megabytes to bytes
-			maxFileSize = maxFileSize * 1024 * 1024;
-			if(file.length() != 0 && file.length() < maxFileSize ){
-				fileOk = true;
+
+			String message=null;
+			if(isDouble){
+				message = String.format("%d/%d;%s;%s",taskID,totalTasks,file.getName(),"Skipped: resource already stored;");
 			}else{
-				String message = "Could not process " + file.getName() + " it has an unacceptable file size.\n\tFile is " + file.length() + " bytes. \n\tShould be more than zero and smaller than " + Config.getInt(Key.MAX_FILE_SIZE) + " bytes ).";
-				LOG.warning(message);
-				System.out.println(message);
+				double durationInSeconds = strategy.store(file.getAbsolutePath(), file.getName());
+				double cpuSecondsPassed = w.timePassed(TimeUnit.SECONDS);
+				String audioDuration = StopWatch.toTime("", (int) Math.round(durationInSeconds));
+				String cpuTimeDuration = w.formattedToString();
+				double timeRatio = durationInSeconds/cpuSecondsPassed;
+				message = String.format("%d/%d;%s;%s;%s;%.2f",taskID,totalTasks,file.getName(),audioDuration,cpuTimeDuration,timeRatio);
 			}
-			return fileOk;
+			LOG.info(message);
+			System.out.println(message);
+
 		}
+
+
 	}