First draft for a buffered writing of files

gama.core/src/gama/core/common/interfaces/IKeyword.java

@@ -152,6 +152,8 @@ public interface IKeyword {
 
 	/** The browse. */
 	String BROWSE = "browse";
+
+	String BUFFERING = "buffering";
 
 	/** The camera. */
 	String CAMERA = "camera";

gama.core/src/gama/core/common/interfaces/ISaveDelegate.java

@@ -19,6 +19,7 @@
 
 import gama.core.runtime.IScope;
 import gama.gaml.expressions.IExpression;
+import gama.gaml.statements.save.SaveOptions;
 import gama.gaml.types.IType;
 import gama.gaml.types.Types;
 
@@ -50,8 +51,7 @@ public interface ISaveDelegate {
 	 *            value> or a list<string>.
 	 * @throws IOException
 	 */
-	void save(IScope scope, IExpression item, File file, String code, boolean addHeader, String type,
-			Object attributesToSave) throws IOException;
+	void save(IScope scope, IExpression item, File file, SaveOptions saveOptions) throws IOException;
 
 	/**
 	 * The type of the item. Returns the gaml type required for triggering this save delegate. If no type is declared

gama.core/src/gama/core/common/preferences/GamaPreferences.java

@@ -45,6 +45,7 @@
 import gama.gaml.compilation.GAML;
 import gama.gaml.compilation.kernel.GamaMetaModel;
 import gama.gaml.operators.Strings;
+import gama.gaml.statements.SaveStatement;
 import gama.gaml.types.IType;
 import one.util.streamex.StreamEx;
 
@@ -76,6 +77,8 @@ public class GamaPreferences {
 			() -> GamaColor.get(199, 234, 229), () -> GamaColor.get(128, 205, 193), () -> GamaColor.get(53, 151, 143),
 			() -> GamaColor.get(1, 102, 94), () -> GamaColor.get(0, 60, 48) };
 
+	public static final String PREF_BUFFERING_STRATEGY = "pref_buffering_strategy";
+
 	/**
 	 *
 	 * Interface tab
@@ -864,6 +867,12 @@ public static class External {
 				"In-memory shapefile mapping (optimizes access to shapefile data in exchange for increased memory usage)",
 				true, IType.BOOL, true).in(NAME, OPTIMIZATIONS);
 
+		/** The Constant DEFAULT_BUFFERING_STRATEGY. */
+		public static final Pref<String> DEFAULT_BUFFERING_STRATEGY =
+				create(PREF_BUFFERING_STRATEGY, "Default buffering strategy for save statement", SaveStatement.NO_BUFFERING, IType.STRING, true)
+				.among(SaveStatement.BUFFERING_STRATEGIES.stream().toList())
+				.in(NAME, OPTIMIZATIONS);
+
 		/**
 		 * Paths to libraries
 		 */

gama.core/src/gama/core/kernel/simulation/SimulationAgent.java

@@ -406,6 +406,7 @@ protected void postStep(final IScope scope) {
 		executer.executeOneShotActions();
 		if (outputs != null) { outputs.step(this.getScope()); }
 		ownClock.step();
+		GAMA.flushWriteStep(this);
 	}
 
 	@Override
@@ -437,8 +438,8 @@ public Object _init_(final IScope scope) {
 	public void dispose() {
 		if (dead) return;
 		executer.executeDisposeActions();
-		// hqnghi if simulation come from popultion extern, dispose pop first
-		// and then their outputs
+		// hqnghi if simulation comes from an external population, dispose this population first
+		// and then its outputs
 
 		if (externMicroPopulations != null) { externMicroPopulations.clear(); }
 
@@ -455,6 +456,9 @@ public void dispose() {
 			}
 		}
 		if (externalInitsAndParameters != null) { externalInitsAndParameters.clear(); }
+
+		//we make sure that all pending write operations are flushed
+		GAMA.flushWriteSimulation(this);
 		GAMA.releaseScope(getScope());
 		// scope = null;
 		super.dispose();

gama.core/src/gama/core/runtime/GAMA.java

@@ -9,6 +9,7 @@
  ********************************************************************************************************/
 package gama.core.runtime;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -35,12 +36,14 @@
 import gama.core.runtime.IExperimentStateListener.State;
 import gama.core.runtime.benchmark.Benchmark;
 import gama.core.runtime.benchmark.StopWatch;
+import gama.core.runtime.concurrent.WriteController;
 import gama.core.runtime.exceptions.GamaRuntimeException;
 import gama.core.runtime.exceptions.GamaRuntimeException.GamaRuntimeFileException;
 import gama.dev.DEBUG;
 import gama.gaml.compilation.ISymbol;
 import gama.gaml.compilation.kernel.GamaBundleLoader;
 import gama.gaml.compilation.kernel.GamaMetaModel;
+import gama.gaml.statements.save.SaveOptions;
 
 /**
  * Written by drogoul Modified on 23 nov. 2009
@@ -98,6 +101,22 @@ public class GAMA {
 	// hqnghi: add several controllers to have multi-thread experiments
 	private static final List<IExperimentController> controllers = new CopyOnWriteArrayList<>();
 
+	private static final WriteController writeController = new WriteController();
+
+	public static boolean askWriteFile(SimulationAgent owner, File f, String content, final SaveOptions options) {
+		return writeController.askWrite(f.getAbsolutePath(), owner, content, options);
+	}
+	public static boolean askWriteFile(SimulationAgent owner, File f, CharSequence content, final SaveOptions options) {
+		return writeController.askWrite(f.getAbsolutePath(), owner, content, options);
+	}
+
+	public static boolean flushWriteSimulation(SimulationAgent owner) {
+		return writeController.flushSimulationOwner(owner);
+	}
+	public static boolean flushWriteStep(SimulationAgent owner) {
+		return writeController.flushCycleOwner(owner);
+	}
+
 	/**
 	 * Gets the controllers.
 	 *

gama.core/src/gama/core/runtime/concurrent/WriteController.java

@@ -0,0 +1,157 @@
+package gama.core.runtime.concurrent;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.NotImplementedException;
+
+import gama.core.kernel.simulation.SimulationAgent;
+import gama.core.runtime.GAMA;
+import gama.core.runtime.exceptions.GamaRuntimeException;
+import gama.gaml.statements.save.SaveOptions;
+
+public class WriteController {
+
+	public enum BufferingStrategies{
+		NO_BUFFERING,
+		PER_CYCLE_BUFFERING,
+		PER_SIMULATION_BUFFERING
+	}
+
+
+	public class WriteTask {
+		final public StringBuilder content;
+		final public Charset encoding;
+		protected boolean rewrite;
+
+		public WriteTask(final CharSequence initialContent, final Charset encodingType, final boolean rewriteFile) {
+			content = new StringBuilder(initialContent);
+			encoding = encodingType;
+			rewrite = rewriteFile;
+		}
+
+		public void setRewrite(boolean rewrite) {
+			this.rewrite = rewrite;
+		}
+		public boolean isRewriting() {
+			return rewrite;
+		}
+	}
+
+	protected Map<String, Map<SimulationAgent, WriteTask>> fileWritingPerSimulationMap;
+	protected Map<String, Map<SimulationAgent, WriteTask>> fileWritingPerCycleMap;
+
+	public WriteController() {
+		fileWritingPerSimulationMap = new HashMap<>();
+		fileWritingPerCycleMap = new HashMap<>();
+	}
+
+	public boolean askWrite(final String fileId, final SimulationAgent owner, final CharSequence content, final SaveOptions options) {
+		switch (options.bufferingStrategy) {
+			case PER_SIMULATION_BUFFERING:
+				return appendWriteSimulation(fileId, owner, content, options);
+			case PER_CYCLE_BUFFERING:
+				return appendWriteCycle(fileId, owner, content, options);
+			case NO_BUFFERING:
+				return directWrite(fileId, content, options.getCharset(), !options.rewrite);
+			default:
+				throw GamaRuntimeException.create(new NotImplementedException("This buffering strategie has not been implemented yet: " + options.bufferingStrategy.toString()), owner.getScope());
+		}
+	}
+
+	protected boolean appendWriteRequestToMap(final String fileId,final  SimulationAgent owner,final  CharSequence content,final  Map<String, Map<SimulationAgent, WriteTask>> map, final SaveOptions options) {
+		// If we don't have any map for this file yet we create one
+		Map<SimulationAgent, WriteTask> fileSavingAsksMap = map.get(fileId);
+		if (fileSavingAsksMap == null) {
+			fileSavingAsksMap = new HashMap<>();
+			map.put(fileId, fileSavingAsksMap);
+		}
+
+		// We look up for the previous write request of the owner simulation in the map
+		// if there's already one we append our content or rewrite, depending on the append parameter
+		// else we create one with the content as its initial value
+		WriteTask askRequest = fileSavingAsksMap.get(owner);
+		if (askRequest == null) {
+			try {
+				fileSavingAsksMap.put(owner, new WriteTask(content, options.getCharset(), options.rewrite));	
+				return true;
+			}
+			catch(Exception ex) {
+				GAMA.reportError(owner.getScope(), GamaRuntimeException.create(ex, owner.getScope()), false);
+				return false;
+			}
+		}
+		else {
+			// If we are not in append mode, we empty the buffer
+			if (options.rewrite) {
+				askRequest.setRewrite(true);
+				askRequest.content.setLength(0);
+			}
+			askRequest.content.append(content);
+			return true;
+		}
+	}
+
+	protected boolean appendWriteSimulation(final String fileId, final SimulationAgent owner, final CharSequence content, final SaveOptions options) {
+		return appendWriteRequestToMap(fileId, owner, content, fileWritingPerSimulationMap, options);
+	}
+
+	protected boolean appendWriteCycle(final String fileId, final SimulationAgent owner, final CharSequence content, final SaveOptions options) {
+		return appendWriteRequestToMap(fileId, owner, content, fileWritingPerCycleMap, options);
+	}
+	protected boolean directWrite(final String fileId, final CharSequence content, final Charset charset, final boolean append) {
+		try (FileWriter fr = new FileWriter(new File(fileId), charset, append)){
+			fr.append(content);
+			fr.flush();
+			fr.close();
+			return true;
+		} catch (IOException e) {
+			e.printStackTrace();
+			return false;
+		}
+	}
+
+	/**
+	 * Flushes all the save requests made by a simulation saved in a map
+	 * @param owner: the simulation in which the save statements have been executed
+	 * @return true if everything went well, false in case of error
+	 */
+	protected boolean flushMapOwner(SimulationAgent owner, Map<String, Map<SimulationAgent, WriteTask>> map) {
+		boolean success = true;
+		for(var entry : map.entrySet()) {
+			var writeTask = entry.getValue().get(owner);
+			if (writeTask != null) {
+				var writeSuccess = directWrite(entry.getKey(), writeTask.content, writeTask.encoding, !writeTask.rewrite);
+				// we don't return false directly because we try to flush as much files as possible
+				success &= writeSuccess;
+				// if the write was successful we remove the operation from the map 
+				if (writeSuccess) {
+					entry.getValue().remove(owner);
+				}
+			}
+		}
+		return success;
+	}
+	/**
+	 * Flushes all the save requests made by a simulation with the 'per_simulation_buffering' strategy
+	 * @param owner: the simulation in which the save statements have been executed
+	 * @return true if everything went well, false in case of error
+	 */
+	public boolean flushSimulationOwner(SimulationAgent owner) {
+		return flushMapOwner(owner, fileWritingPerSimulationMap);		
+	}
+
+	/**
+	 * Flushes all the save requests made by a simulation with the 'per_cycle_buffering' strategy
+	 * @param owner: the simulation in which the save statements have been executed
+	 * @return true if everything went well, false in case of error
+	 */
+	public boolean flushCycleOwner(SimulationAgent owner) {
+		return flushMapOwner(owner, fileWritingPerCycleMap);
+	}
+
+}

gama.core/src/gama/gaml/operators/Files.java

@@ -12,8 +12,6 @@
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.file.Path;
@@ -37,9 +35,11 @@
 import gama.annotations.precompiler.ITypeProvider;
 import gama.core.common.interfaces.IKeyword;
 import gama.core.common.util.FileUtils;
+import gama.core.kernel.simulation.SimulationAgent;
 import gama.core.metamodel.agent.IAgent;
 import gama.core.metamodel.shape.GamaShape;
 import gama.core.metamodel.shape.IShape;
+import gama.core.runtime.GAMA;
 import gama.core.runtime.IScope;
 import gama.core.runtime.exceptions.GamaRuntimeException;
 import gama.core.util.IContainer;
@@ -59,16 +59,7 @@
 public class Files {
 
 
-	// @operator (
-	// value = IKeyword.FILE,
-	// can_be_const = true,
-	// category = IOperatorCategory.FILE,
-	// concept = { IConcept.FILE })
-	// @doc (
-	// value = "Creates a file in read/write mode, setting its contents to the container passed in parameter",
-	// comment = "The type of container to pass will depend on the type of file (see the management of files in the
-	// documentation). Can be used to copy files since files are considered as containers. For example: save
-	// file('image_copy.png', file('image.png')); will copy image.png to image_copy.png")
+
 	/**
 	 * From.
 	 *
@@ -89,24 +80,6 @@ public static IGamaFile from(final IScope scope, final String s, final IContaine
 		return (IGamaFile) Types.FILE.cast(scope, s, container, key, content, false);
 	}
 
-	//
-	// @operator (
-	// value = IKeyword.FILE,
-	// can_be_const = true,
-	// category = IOperatorCategory.FILE,
-	// concept = { IConcept.FILE })
-	// @doc (
-	// value = "opens a file in read only mode, creates a GAML file object, and tries to determine and store the file
-	// content in the contents attribute.",
-	// comment = "The file should have a supported extension, see file type definition for supported file extensions.",
-	// usages = @usage ("If the specified string does not refer to an existing file, an exception is risen when the
-	// variable is used."),
-	// examples = { @example (
-	// value = "let fileT type: file value: file(\"../includes/Stupid_Cell.Data\"); "),
-	// @example (
-	// value = " // fileT represents the file \"../includes/Stupid_Cell.Data\""),
-	// @example (
-	// value = " // fileT.contents here contains a matrix storing all the data of the text file") },
 	/**
 	 * From.
 	 *
@@ -753,4 +726,33 @@ public static IGamaFile newFolder(final IScope scope, final String folder) throw
 
 	}
 
+	/**
+	 * Flushes all the pending write operations in the current simulation
+	 * @param scope
+	 * @return true if everything went well, false if there was a problem while flushing
+	 * @throws GamaRuntimeException
+	 */
+	@operator (
+			value = { "flush_all_files" },
+			category = IOperatorCategory.FILE,
+			concept = { IConcept.FILE },
+			type = IType.BOOL
+			)
+	@doc (
+			value = "Flushes all the pending write operations in the current simulation. This operator is only useful "
+					+ "in simulation that save files using a buffering strategy.",
+			comment = "",
+			usages = {
+					@usage ("If the specified string does not refer to an existing repository, the repository is created."),
+					@usage ("If the string refers to an existing file, an exception is risen.") },
+			examples = {
+					@example ("file dirNewT <- new_folder(\"incl/\");   	// dirNewT represents the repository \"../incl/\""),
+					@example ("															// eventually creates the directory ../incl") },
+			see = { "save"})
+	public static boolean flushAllFiles(final IScope scope, final SimulationAgent simulation) throws GamaRuntimeException {
+		boolean success = GAMA.flushWriteStep(simulation);
+		success &= GAMA.flushWriteSimulation(simulation);
+		return success;
+	}
+
 }