diff --git a/gama.core/src/gama/core/common/interfaces/IKeyword.java b/gama.core/src/gama/core/common/interfaces/IKeyword.java index 634281e406..6483083722 100644 --- a/gama.core/src/gama/core/common/interfaces/IKeyword.java +++ b/gama.core/src/gama/core/common/interfaces/IKeyword.java @@ -152,6 +152,8 @@ public interface IKeyword { /** The browse. */ String BROWSE = "browse"; + + String BUFFERING = "buffering"; /** The camera. */ String CAMERA = "camera"; @@ -299,6 +301,8 @@ public interface IKeyword { /** The enables. */ String ENABLES = "enables"; + + String END = "end"; /** The enter. */ String ENTER = "enter"; diff --git a/gama.core/src/gama/core/common/interfaces/ISaveDelegate.java b/gama.core/src/gama/core/common/interfaces/ISaveDelegate.java index 6d488ab4c3..cc879034e3 100644 --- a/gama.core/src/gama/core/common/interfaces/ISaveDelegate.java +++ b/gama.core/src/gama/core/common/interfaces/ISaveDelegate.java @@ -19,6 +19,7 @@ import gama.core.runtime.IScope; import gama.gaml.expressions.IExpression; +import gama.gaml.statements.save.SaveOptions; import gama.gaml.types.IType; import gama.gaml.types.Types; @@ -50,8 +51,7 @@ public interface ISaveDelegate { * value> or a list. * @throws IOException */ - void save(IScope scope, IExpression item, File file, String code, boolean addHeader, String type, - Object attributesToSave) throws IOException; + void save(IScope scope, IExpression item, File file, SaveOptions saveOptions) throws IOException; /** * The type of the item. Returns the gaml type required for triggering this save delegate. If no type is declared diff --git a/gama.core/src/gama/core/common/preferences/GamaPreferences.java b/gama.core/src/gama/core/common/preferences/GamaPreferences.java index d9f689f655..c3d3e351d0 100644 --- a/gama.core/src/gama/core/common/preferences/GamaPreferences.java +++ b/gama.core/src/gama/core/common/preferences/GamaPreferences.java @@ -36,6 +36,7 @@ import gama.core.outputs.layers.properties.ICameraDefinition; import gama.core.runtime.GAMA; import gama.core.runtime.PlatformHelper; +import gama.core.runtime.concurrent.BufferingController; import gama.core.util.GamaColor; import gama.core.util.GamaFont; import gama.core.util.GamaMapFactory; @@ -76,6 +77,9 @@ public class GamaPreferences { () -> GamaColor.get(199, 234, 229), () -> GamaColor.get(128, 205, 193), () -> GamaColor.get(53, 151, 143), () -> GamaColor.get(1, 102, 94), () -> GamaColor.get(0, 60, 48) }; + public static final String PREF_SAVE_BUFFERING_STRATEGY = "pref_save_buffering_strategy"; + public static final String PREF_WRITE_BUFFERING_STRATEGY = "pref_write_buffering_strategy"; + /** * * Interface tab @@ -864,6 +868,17 @@ public static class External { "In-memory shapefile mapping (optimizes access to shapefile data in exchange for increased memory usage)", true, IType.BOOL, true).in(NAME, OPTIMIZATIONS); + /** The Constant DEFAULT_BUFFERING_STRATEGY. */ + public static final Pref DEFAULT_SAVE_BUFFERING_STRATEGY = + create(PREF_SAVE_BUFFERING_STRATEGY, "Default buffering strategy for the save statement", BufferingController.NO_BUFFERING, IType.STRING, true) + .among(BufferingController.BUFFERING_STRATEGIES.stream().toList()) + .in(NAME, OPTIMIZATIONS); + + public static final Pref DEFAULT_WRITE_BUFFERING_STRATEGY = + create(PREF_WRITE_BUFFERING_STRATEGY, "Default buffering strategy for the write statement", BufferingController.NO_BUFFERING, IType.STRING, true) + .among(BufferingController.BUFFERING_STRATEGIES.stream().toList()) + .in(NAME, OPTIMIZATIONS); + /** * Paths to libraries */ diff --git a/gama.core/src/gama/core/kernel/simulation/SimulationAgent.java b/gama.core/src/gama/core/kernel/simulation/SimulationAgent.java index add777be45..2c3dd2e9cf 100644 --- a/gama.core/src/gama/core/kernel/simulation/SimulationAgent.java +++ b/gama.core/src/gama/core/kernel/simulation/SimulationAgent.java @@ -406,6 +406,8 @@ protected void postStep(final IScope scope) { executer.executeOneShotActions(); if (outputs != null) { outputs.step(this.getScope()); } ownClock.step(); + GAMA.flushSaveFileStep(this); + GAMA.flushWriteStep(this); } @Override @@ -437,8 +439,8 @@ public Object _init_(final IScope scope) { public void dispose() { if (dead) return; executer.executeDisposeActions(); - // hqnghi if simulation come from popultion extern, dispose pop first - // and then their outputs + // hqnghi if simulation comes from an external population, dispose this population first + // and then its outputs if (externMicroPopulations != null) { externMicroPopulations.clear(); } @@ -455,6 +457,10 @@ public void dispose() { } } if (externalInitsAndParameters != null) { externalInitsAndParameters.clear(); } + + //we make sure that all pending write operations are flushed + GAMA.flushSaveFilePerOwner(this); + GAMA.flushWritePerAgent(this); GAMA.releaseScope(getScope()); // scope = null; super.dispose(); diff --git a/gama.core/src/gama/core/metamodel/agent/AbstractAgent.java b/gama.core/src/gama/core/metamodel/agent/AbstractAgent.java index 3ddd53ced2..2c00c05923 100644 --- a/gama.core/src/gama/core/metamodel/agent/AbstractAgent.java +++ b/gama.core/src/gama/core/metamodel/agent/AbstractAgent.java @@ -129,6 +129,8 @@ public void dispose() { attributes.clear(); attributes = null; } + GAMA.flushSaveFilePerOwner(this); + GAMA.flushWritePerAgent(this); } /** diff --git a/gama.core/src/gama/core/runtime/GAMA.java b/gama.core/src/gama/core/runtime/GAMA.java index ce0e8fcbb8..1063b900c2 100644 --- a/gama.core/src/gama/core/runtime/GAMA.java +++ b/gama.core/src/gama/core/runtime/GAMA.java @@ -9,6 +9,7 @@ ********************************************************************************************************/ package gama.core.runtime; +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -32,15 +33,20 @@ import gama.core.kernel.model.IModel; import gama.core.kernel.root.PlatformAgent; import gama.core.kernel.simulation.SimulationAgent; +import gama.core.metamodel.agent.AbstractAgent; import gama.core.runtime.IExperimentStateListener.State; import gama.core.runtime.benchmark.Benchmark; import gama.core.runtime.benchmark.StopWatch; +import gama.core.runtime.concurrent.BufferingController; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.runtime.exceptions.GamaRuntimeException.GamaRuntimeFileException; +import gama.core.util.GamaColor; import gama.dev.DEBUG; import gama.gaml.compilation.ISymbol; import gama.gaml.compilation.kernel.GamaBundleLoader; import gama.gaml.compilation.kernel.GamaMetaModel; +import gama.gaml.statements.save.SaveOptions; /** * Written by drogoul Modified on 23 nov. 2009 @@ -98,6 +104,31 @@ public class GAMA { // hqnghi: add several controllers to have multi-thread experiments private static final List controllers = new CopyOnWriteArrayList<>(); + private static final BufferingController bufferingController = new BufferingController(); + + + + public static boolean askWriteFile(final IScope scope, final File f, final CharSequence content, final SaveOptions options) { + return bufferingController.askWriteFile(f.getAbsolutePath(), scope, content, options); + } + public static boolean askWriteConsole(final IScope scope, final StringBuilder content, final GamaColor color, final BufferingStrategies strategy) { + return bufferingController.askWriteConsole(scope, content, color, strategy); + } + + public static boolean flushSaveFilePerOwner(final AbstractAgent owner) { + return bufferingController.flushSaveFilesOfOwner(owner); + } + public static boolean flushSaveFileStep(final SimulationAgent owner) { + return bufferingController.flushSaveFilesInCycle(owner); + } + public static void flushWriteStep(final SimulationAgent owner) { + bufferingController.flushWriteInCycle(owner); + } + public static void flushWritePerAgent(final AbstractAgent owner) { + bufferingController.flushWriteOfOwner(owner); + } + + /** * Gets the controllers. * diff --git a/gama.core/src/gama/core/runtime/concurrent/BufferingController.java b/gama.core/src/gama/core/runtime/concurrent/BufferingController.java new file mode 100644 index 0000000000..a5a2cb3467 --- /dev/null +++ b/gama.core/src/gama/core/runtime/concurrent/BufferingController.java @@ -0,0 +1,363 @@ +package gama.core.runtime.concurrent; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.NotImplementedException; + +import gama.core.metamodel.agent.AbstractAgent; +import gama.core.runtime.GAMA; +import gama.core.runtime.IScope; +import gama.core.runtime.exceptions.GamaRuntimeException; +import gama.core.util.GamaColor; +import gama.gaml.statements.save.SaveOptions; + +public class BufferingController { + + + public static final String PER_CYCLE_BUFFERING = "per_cycle"; + public static final String PER_SIMULATION_BUFFERING = "per_simulation"; + public static final String PER_AGENT = "per_agent"; + public static final String NO_BUFFERING = "no_buffering"; + + public static final Set BUFFERING_STRATEGIES = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(PER_CYCLE_BUFFERING, PER_SIMULATION_BUFFERING, NO_BUFFERING, PER_AGENT))); + + + public enum BufferingStrategies{ + NO_BUFFERING, + PER_CYCLE_BUFFERING, + PER_SIMULATION_BUFFERING, + PER_AGENT, + } + + + /** + * Converts a string into a BufferingStrategies if it matches the corresponding static variables. If not it returns NO_BUFFERING + * @param s + * @return + */ + public static BufferingStrategies stringToBufferingStrategies(IScope scope, String s) { + switch (s){ + case BufferingController.PER_CYCLE_BUFFERING: + return BufferingStrategies.PER_CYCLE_BUFFERING; + case BufferingController.PER_SIMULATION_BUFFERING: + return BufferingStrategies.PER_SIMULATION_BUFFERING; + case BufferingController.NO_BUFFERING: + return BufferingStrategies.NO_BUFFERING; + case BufferingController.PER_AGENT: + return BufferingStrategies.PER_AGENT; + default: + throw GamaRuntimeException.create(new NotImplementedException("This buffering strategie has not been implemented yet: " + s), scope); + } + } + + + public static class TextBuffer { + + final public StringBuilder content; + final public Charset encoding; + final public GamaColor color; + protected boolean rewrite; + + // Constructor for a text buffer on a file + public TextBuffer(final CharSequence initialContent, final Charset encodingType, final boolean rewriteFile) { + content = new StringBuilder(initialContent); + encoding = encodingType; + rewrite = rewriteFile; + color = null; + } + + // Constructor for a text buffer on the console + public TextBuffer(final CharSequence initialContent, final GamaColor textColor) { + content = new StringBuilder(initialContent); + color = textColor; + encoding = null; + rewrite = false; + } + + public void setRewrite(boolean rewrite) { + this.rewrite = rewrite; + } + public boolean isRewriting() { + return rewrite; + } + } + + protected Map> fileBufferPerAgent; + protected Map> fileBufferPerAgentForCycles; + protected Map> consoleBufferListPerAgent; + protected Map> consoleBufferListPerAgentForCycles; + + public BufferingController() { + fileBufferPerAgent = new HashMap<>(); + fileBufferPerAgentForCycles = new HashMap<>(); + consoleBufferListPerAgent = new HashMap<>(); + consoleBufferListPerAgentForCycles = new HashMap<>(); + } + + /** + * Ask to write on a file following a given buffering strategy and some saving options. + * This method will take care of creating or not a new buffer if needed. + * If the strategy is different from no_buffering, the text will only be written when the corresponding + * flush operation is called on the file or on the agent. This is automatically managed inside the agents. + * If multiple calls of that method for the same file are made before flushing, + * the texts will be concatenated, resulting in performance gains. + * If the strategy is no_buffering, then the file is directly written. + * @param fileId The id used to represent the file, it should be its absolute path + * @param scope the scope from which the save statement was called, used to identify the "owner" of the request + * @param content the text to write in the file + * @param options the saving options (rewrite, buffering strategy etc. ) + * @return true if the operation was successful, false otherwise + */ + public boolean askWriteFile(final String fileId, final IScope scope, final CharSequence content, final SaveOptions options) { + AbstractAgent owner = scope.getSimulation(); + switch (options.bufferingStrategy) { + case PER_AGENT, PER_SIMULATION_BUFFERING: + // in case it's per agent we just switch the owner to the calling agent + // instead of the whole simulation + if (options.bufferingStrategy == BufferingStrategies.PER_AGENT) { + owner = (AbstractAgent) scope.getAgent(); + } + return appendSaveFileRequestToMap(owner, getOrInitBufferingMap(fileId, fileBufferPerAgent), content, options); + case PER_CYCLE_BUFFERING: + return appendSaveFileRequestToMap(owner, getOrInitBufferingMap(fileId, fileBufferPerAgentForCycles), content, options); + case NO_BUFFERING: + return directWriteFile(fileId, content, options.getCharset(), !options.rewrite); + default: + throw GamaRuntimeException.create(new NotImplementedException("This buffering strategie has not been implemented yet: " + options.bufferingStrategy.toString()), owner.getScope()); + } + } + + /** + * Ask to write some text in the console following a given buffering strategy and with a given color. + * This method will take care of creating or not a new buffer if needed. + * If the strategy is different from no_buffering, the text will only be written when the corresponding + * flush operation is called. This is automatically managed inside the agents. + * If multiple calls of that method are made before flushing, + * the texts will be concatenated, resulting in performance gains. + * If the strategy is no_buffering, then the content is directly written in the console. + * @param scope the scope from which the write statement was called + * @param content the text to print + * @param color the color of the text + * @param bufferingStrategy the buffering strategy to apply + * @return true if the operation is successful, false otherwise + */ + public boolean askWriteConsole(final IScope scope, final StringBuilder content, final GamaColor color, final BufferingStrategies bufferingStrategy) { + AbstractAgent owner = scope.getSimulation(); + switch (bufferingStrategy) { + case PER_AGENT, PER_SIMULATION_BUFFERING: + // in case it's per agent we just switch the owner to the calling agent + // instead of the whole simulation + if (bufferingStrategy == BufferingStrategies.PER_AGENT) { + owner = (AbstractAgent) scope.getAgent(); + } + return appendWriteConsoleRequestToMap(owner, consoleBufferListPerAgent, content, color); + case PER_CYCLE_BUFFERING: + return appendWriteConsoleRequestToMap(owner, consoleBufferListPerAgentForCycles, content, color); + case NO_BUFFERING: + scope.getGui().getConsole().informConsole(content.toString(), scope.getRoot(), color); + return true; + default: + throw GamaRuntimeException.create(new NotImplementedException("This buffering strategie has not been implemented yet: " + bufferingStrategy.toString()), owner.getScope()); + } + } + + /** + * Tries to get the existing map of agent/buffer for one given file. If it doesn't exist it will + * be created and added to the map in which it was looking in. + * @param fileId the id of the file, it should be its absolute path + * @param map the map in which to look in + * @return the corresponding map of agent/buffer + */ + protected Map getOrInitBufferingMap(final String fileId, final Map> map){ + // If we don't have any map for this file yet we create one + Map bufferingMap = map.get(fileId); + if (bufferingMap == null) { + bufferingMap = new HashMap<>(); + map.put(fileId, bufferingMap); + } + return bufferingMap; + } + + /** + * Takes care of properly adding the content of a file saving request into the map. + * It will create a buffer if none exists or append the content if one is already present. + * It will also take care of resetting the content in case the new request has the rewrite option set to true + * @param owner the agent that is responsible for asking to write, will be used later to flush + * @param bufferingMap the map containing already present buffers + * @param content the text to write + * @param options the saving options (used to get the rewrite option) + * @return true if the operation was successful, false otherwise + */ + protected boolean appendSaveFileRequestToMap(final AbstractAgent owner, final Map bufferingMap, final CharSequence content, final SaveOptions options) { + + // We look up for the previous request of the owner simulation in the map + // if there's already one we append our content or rewrite, depending on the append parameter + // else we create one with the content as its initial value + TextBuffer request = bufferingMap.get(owner); + if (request == null) { + try { + bufferingMap.put(owner, new TextBuffer(content, options.getCharset(), options.rewrite)); + return true; + } + catch(Exception ex) { + GAMA.reportError(owner.getScope(), GamaRuntimeException.create(ex, owner.getScope()), false); + return false; + } + } + else { + // If we are not in append mode, we empty the buffer + if (options.rewrite) { + request.setRewrite(true); + request.content.setLength(0); + } + request.content.append(content); + return true; + } + } + + /** + * Takes care of properly adding the content of a console write request into the map. + * It will create a buffer if none exists or append the content into the last one if it is compatible (same color). + * @param owner the agent that is responsible for asking to write, will be used later to flush + * @param bufferingMap the map containing the list of already created buffers + * @param content the text to write + * @param color the color in which the text should be printed + * @return true if the operation was successful, false otherwise + */ + protected boolean appendWriteConsoleRequestToMap(final AbstractAgent owner, final Map> bufferingMap, final StringBuilder content, final GamaColor color) { + + // We look up for the previous request of the owner simulation in the map + List requests = bufferingMap.get(owner); + + // If there's no list yet we create one and add it to the map + if (requests == null) { + requests = new ArrayList(); + bufferingMap.put(owner, requests); + } + + // If the last element of the list is not of the same color as the currently requested color we append a new task with the new color + if (requests.size() == 0 || (requests.get(requests.size()-1).color != null && !requests.get(requests.size()-1).color.equals(color))) { + try { + requests.add(new TextBuffer(content, color)); + return true; + } + catch(Exception ex) { + GAMA.reportError(owner.getScope(), GamaRuntimeException.create(ex, owner.getScope()), false); + return false; + } + } + else { + requests.get(requests.size()-1).content.append(content); + return true; + } + } + + + /** + * Creates a file object and directly writes the content into it. No buffering. + * @param fileId the path of the file + * @param content the text to print + * @param charset the charset used to write + * @param append if true the content will be appened, else it will replace the current file content (if any) + * @return true if no exception, false otherwise + */ + protected boolean directWriteFile(final String fileId, final CharSequence content, final Charset charset, final boolean append) { + try { + FileUtils.write(new File(fileId), content, charset, append); + return true; + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + + + + /** + * Flushes all the save requests linked to an agent (per_agent or per_simulation buffering) + * @param owner the agent in which the save statements have been executed + * @param map the map in which to look up + * @return true if everything went well, false in case of error + */ + protected boolean flushAllFilesOfOwner(AbstractAgent owner, Map> map) { + boolean success = true; + for(var entry : map.entrySet()) { + var writeTask = entry.getValue().get(owner); + if (writeTask != null) { + var writeSuccess = directWriteFile(entry.getKey(), writeTask.content, writeTask.encoding, !writeTask.rewrite); + // we don't return false directly because we try to flush as much files as possible + success &= writeSuccess; + // if the write was successful we remove the operation from the map + if (writeSuccess) { + entry.getValue().remove(owner); + } + } + } + return success; + } + + + /** + * Flushes all the write requests linked to an agent (per_agent or per_simulation buffering) + * @param owner the agent in which the write statements have been executed + * @param map the map in which to look up + */ + protected void flushAllWriteOfOwner(final AbstractAgent owner, final Map> map) { + var tasks = map.get(owner); + if (tasks != null) { + var scope = owner.getScope(); + for (var task : tasks) { + scope.getGui().getConsole().informConsole(task.content.toString(), scope.getRoot(), task.color); + } + tasks.clear(); + } + } + + + + /** + * Flushes all the save requests made by an agent with the 'per_simulation' or 'per_agent' strategy + * @param owner the simulation or agent in which the save statements have been executed + * @return true if everything went well, false in case of error + */ + public boolean flushSaveFilesOfOwner(AbstractAgent owner) { + return flushAllFilesOfOwner(owner, fileBufferPerAgent); + } + + /** + * Flushes all the save requests made in a simulation with the 'per_cycle' strategy + * @param owner the simulation in which the save statements have been executed + * @return true if everything went well, false in case of error + */ + public boolean flushSaveFilesInCycle(AbstractAgent owner) { + return flushAllFilesOfOwner(owner, fileBufferPerAgentForCycles); + } + + /** + * Flushes all the write statement requests made in a simulation with the 'per_cycle' strategy + * @param owner the simulation in which the write statements have been executed + */ + public void flushWriteInCycle(AbstractAgent owner) { + flushAllWriteOfOwner(owner, consoleBufferListPerAgentForCycles); + } + + /** + * Flushes all the write requests made by an agent with the 'per_simulation' or 'per_agent' strategy + * @param owner: the agent or simulation in which the write statements have been executed + */ + public void flushWriteOfOwner(AbstractAgent owner) { + flushAllWriteOfOwner(owner, consoleBufferListPerAgent); + } + +} diff --git a/gama.core/src/gama/gaml/operators/Files.java b/gama.core/src/gama/gaml/operators/Files.java index 3b6649d0ea..c2db6dca1c 100644 --- a/gama.core/src/gama/gaml/operators/Files.java +++ b/gama.core/src/gama/gaml/operators/Files.java @@ -12,8 +12,6 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Path; @@ -37,9 +35,11 @@ import gama.annotations.precompiler.ITypeProvider; import gama.core.common.interfaces.IKeyword; import gama.core.common.util.FileUtils; +import gama.core.kernel.simulation.SimulationAgent; import gama.core.metamodel.agent.IAgent; import gama.core.metamodel.shape.GamaShape; import gama.core.metamodel.shape.IShape; +import gama.core.runtime.GAMA; import gama.core.runtime.IScope; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.IContainer; @@ -59,16 +59,7 @@ public class Files { - // @operator ( - // value = IKeyword.FILE, - // can_be_const = true, - // category = IOperatorCategory.FILE, - // concept = { IConcept.FILE }) - // @doc ( - // value = "Creates a file in read/write mode, setting its contents to the container passed in parameter", - // comment = "The type of container to pass will depend on the type of file (see the management of files in the - // documentation). Can be used to copy files since files are considered as containers. For example: save - // file('image_copy.png', file('image.png')); will copy image.png to image_copy.png") + /** * From. * @@ -89,24 +80,6 @@ public static IGamaFile from(final IScope scope, final String s, final IContaine return (IGamaFile) Types.FILE.cast(scope, s, container, key, content, false); } - // - // @operator ( - // value = IKeyword.FILE, - // can_be_const = true, - // category = IOperatorCategory.FILE, - // concept = { IConcept.FILE }) - // @doc ( - // value = "opens a file in read only mode, creates a GAML file object, and tries to determine and store the file - // content in the contents attribute.", - // comment = "The file should have a supported extension, see file type definition for supported file extensions.", - // usages = @usage ("If the specified string does not refer to an existing file, an exception is risen when the - // variable is used."), - // examples = { @example ( - // value = "let fileT type: file value: file(\"../includes/Stupid_Cell.Data\"); "), - // @example ( - // value = " // fileT represents the file \"../includes/Stupid_Cell.Data\""), - // @example ( - // value = " // fileT.contents here contains a matrix storing all the data of the text file") }, /** * From. * @@ -753,4 +726,33 @@ public static IGamaFile newFolder(final IScope scope, final String folder) throw } + /** + * Flushes all the pending save operations in the current simulation + * @param scope + * @return true if everything went well, false if there was a problem while flushing + * @throws GamaRuntimeException + */ + @operator ( + value = { "flush_all_files" }, + category = IOperatorCategory.FILE, + concept = { IConcept.FILE }, + type = IType.BOOL + ) + @doc ( + value = "Flushes all the pending save operations in the current simulation. ", + comment = "", + usages = { + @usage ("This operator is only useful in simulations that save files using a buffering strategy."), + @usage ("If a file writing fails it returns false, else it returns true."), + @usage ("If a file writing fails it still tries to write the others."), + }, + examples = { + @example ("full_all_files(simulation); // simulation is the current simulation, this can be important to differentiate in case of multi-simulation experiments")}, + see = { "save"}) + public static boolean flushAllFiles(final IScope scope, final SimulationAgent simulation) throws GamaRuntimeException { + boolean success = GAMA.flushSaveFileStep(simulation); + success &= GAMA.flushSaveFilePerOwner(simulation); + return success; + } + } diff --git a/gama.core/src/gama/gaml/statements/SaveStatement.java b/gama.core/src/gama/gaml/statements/SaveStatement.java index 81ed4fa0d6..619882af50 100644 --- a/gama.core/src/gama/gaml/statements/SaveStatement.java +++ b/gama.core/src/gama/gaml/statements/SaveStatement.java @@ -32,8 +32,11 @@ import gama.annotations.precompiler.ISymbolKind; import gama.core.common.interfaces.IKeyword; import gama.core.common.interfaces.ISaveDelegate; +import gama.core.common.preferences.GamaPreferences; import gama.core.common.util.FileUtils; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.IModifiableContainer; import gama.core.util.file.GamaFile.FlushBufferException; @@ -50,6 +53,7 @@ import gama.gaml.interfaces.IGamlIssue; import gama.gaml.operators.Cast; import gama.gaml.statements.SaveStatement.SaveValidator; +import gama.gaml.statements.save.SaveOptions; import gama.gaml.types.GamaFileType; import gama.gaml.types.IType; import gama.gaml.types.Types; @@ -109,6 +113,17 @@ optional = true, doc = @doc ( value = "Allows to specify the attributes of a shape file or GeoJson file where agents are saved. Can be expressed as a list of string or as a literal map. When expressed as a list, each value should represent the name of an attribute of the shape or agent. The keys of the map are the names of the attributes that will be present in the file, the values are whatever expressions neeeded to define their value. ")), + @facet ( + name = IKeyword.BUFFERING, + type = { IType.STRING}, + optional = true, + doc = @doc ( + value = "Allows to specify a buffering strategy to write the file. Accepted values are `" + BufferingController.PER_CYCLE_BUFFERING +"` and `" + BufferingController.PER_SIMULATION_BUFFERING + "`, `" + BufferingController.NO_BUFFERING + "`. " + + "In the case of `"+ BufferingController.PER_CYCLE_BUFFERING +"` or `"+ BufferingController.PER_SIMULATION_BUFFERING +"`, all the write operations in the simulation which used these values would be " + + "executed all at once at the end of the cycle or simulation while keeping the initial order. In case of '" + BufferingController.PER_AGENT + + "' all operations will be released when the agent is killed (or the simulation ends). Those strategies can be used to optimise a " + + "simulation's execution time on models that extensively write in files. " + + "The `" + BufferingController.NO_BUFFERING + "` (which is the system's default) will directly write into the file.")), }, omissible = IKeyword.DATA) @doc ( @@ -151,7 +166,7 @@ @validator (SaveValidator.class) @SuppressWarnings ({ "rawtypes" }) public class SaveStatement extends AbstractStatementSequence{ - + /** The Constant NON_SAVEABLE_ATTRIBUTE_NAMES. */ public static final Set NON_SAVEABLE_ATTRIBUTE_NAMES = Set.of(IKeyword.PEERS, IKeyword.LOCATION, IKeyword.HOST, IKeyword.AGENTS, IKeyword.MEMBERS, IKeyword.SHAPE); @@ -164,6 +179,7 @@ public class SaveStatement extends AbstractStatementSequence{ /** The Constant SYNONYMS. */ private static final SetMultimap SYNONYMS = TreeMultimap.create(); + /** * @param createExecutableExtension @@ -192,6 +208,7 @@ public static void addDelegate(final ISaveDelegate delegate) { } + /** * The Class SaveValidator. */ @@ -208,6 +225,8 @@ public void validate(final StatementDescription description) { final StatementDescription desc = description; final IExpression att = desc.getFacetExpr(ATTRIBUTES); final IExpressionDescription type = desc.getFacet(FORMAT); + final IExpression bufferingStrategy = desc.getFacetExpr(IKeyword.BUFFERING); + if (type != null) { desc.setFacetExprDescription(FORMAT, type); } final IExpression format = type == null ? null : type.getExpression(); @@ -266,29 +285,34 @@ public void validate(final StatementDescription description) { } + if (bufferingStrategy != null && ! BufferingController.BUFFERING_STRATEGIES.contains(bufferingStrategy.literalValue())) { + desc.error("The value for buffering must be '" + BufferingController.NO_BUFFERING +"', '" + BufferingController.PER_CYCLE_BUFFERING + "', '" + BufferingController.PER_AGENT + "'" + "' or '" + BufferingController.PER_SIMULATION_BUFFERING +"'.", + IGamlIssue.WRONG_TYPE); + } + + // Starting from here we validate the attributes, other validations must be done before + if (att == null) return; + final boolean isMap = att instanceof MapExpression; - if (att != null) { - if (!isMap && !att.getGamlType().isTranslatableInto(Types.LIST.of(Types.STRING))) { - desc.error("attributes must be expressed as a map or as a list", + if (!isMap && !att.getGamlType().isTranslatableInto(Types.LIST.of(Types.STRING))) { + desc.error("attributes must be expressed as a map or as a list", + IGamlIssue.WRONG_TYPE, ATTRIBUTES); + return; + } + if (isMap) { + final MapExpression map = (MapExpression) att; + if (map.getGamlType().getKeyType() != Types.STRING) { + desc.error( + "The type of the keys of the attributes map must be string. These will be used for naming the attributes in the file", IGamlIssue.WRONG_TYPE, ATTRIBUTES); return; } - if (isMap) { - final MapExpression map = (MapExpression) att; - if (map.getGamlType().getKeyType() != Types.STRING) { - desc.error( - "The type of the keys of the attributes map must be string. These will be used for naming the attributes in the file", - IGamlIssue.WRONG_TYPE, ATTRIBUTES); - return; - } - } - - if (ext != null && format == null && !"shp".equals(ext) && !"json".equals(ext) && !"geojson".equals(ext) || format != null - && !"shp".equals(format.literalValue()) && !"geojson".equals(format.literalValue()) && !"json".equals(format.literalValue())) { - desc.warning("Attributes can only be defined for shape, geojson or json files", IGamlIssue.WRONG_TYPE, - ATTRIBUTES); - } + } + if (ext != null && format == null && !"shp".equals(ext) && !"json".equals(ext) && !"geojson".equals(ext) || format != null + && !"shp".equals(format.literalValue()) && !"geojson".equals(format.literalValue()) && !"json".equals(format.literalValue())) { + desc.warning("Attributes can only be defined for shape, geojson or json files", IGamlIssue.WRONG_TYPE, + ATTRIBUTES); } /** The t. */ @@ -297,7 +321,6 @@ public void validate(final StatementDescription description) { /** The species. */ final SpeciesDescription species = t.getSpecies(); - if (att == null) return; if (species == null) { if (isMap) { @@ -308,6 +331,7 @@ public void validate(final StatementDescription description) { // desc.error("Attributes can only be saved for agents", IGamlIssue.UNKNOWN_FACET, // att == null ? WITH : ATTRIBUTES); } + } /** @@ -341,6 +365,8 @@ private boolean areSynonyms(final String ext, final String id) { /** The rewrite expr. */ private final IExpression rewriteExpr; + + private final IExpression bufferingStrategy; /** * Instantiates a new save statement. @@ -355,6 +381,7 @@ public SaveStatement(final IDescription desc) { format = getFacet(IKeyword.FORMAT); rewriteExpr = getFacet(IKeyword.REWRITE); attributesFacet = getFacet(IKeyword.ATTRIBUTES); + bufferingStrategy = getFacet(IKeyword.BUFFERING); } /** @@ -368,21 +395,33 @@ private boolean shouldOverwrite(final IScope scope) { if (rewriteExpr == null) return true; return Cast.asBool(scope, rewriteExpr.value(scope)); } + + /** + * In case the save statement is called with a file object, calls the save method from this object + * @param scope + * @return + */ + protected Object saveFile(IScope scope) { + if (!Types.FILE.isAssignableFrom(item.getGamlType())) return null; + final IGamaFile theFile = (IGamaFile) item.value(scope); + if (theFile != null) { + // Passes directly the facets of the statement, like crs, etc. + theFile.save(scope, description.getFacets()); + } + return theFile; + } @SuppressWarnings ("unchecked") @Override public Object privateExecuteIn(final IScope scope) throws GamaRuntimeException { + // if item is null, there's nothing to write if (item == null) return null; - // First case: we have a file as item; + + // First case: we have no destination file, so it means the item is a file; if (file == null) { - if (!Types.FILE.isAssignableFrom(item.getGamlType())) return null; - final IGamaFile theFile = (IGamaFile) item.value(scope); - if (theFile != null) { - // Passes directly the facets of the statement, like crs, etc. - theFile.save(scope, description.getFacets()); - } - return theFile; + return saveFile(scope); } + final String fileName = Cast.asString(scope, file.value(scope)); final String filePath = FileUtils.constructAbsoluteFilePath(scope, fileName, false); if (filePath == null || "".equals(filePath)) return null; @@ -411,7 +450,6 @@ public Object privateExecuteIn(final IScope scope) throws GamaRuntimeException { } } typeExp = com.google.common.io.Files.getFileExtension(fileName); - } // We may have the case of a string (instead of a literal) @@ -420,14 +458,18 @@ public Object privateExecuteIn(final IScope scope) throws GamaRuntimeException { typeExp = Cast.asString(scope, format.value(scope)); if (!DELEGATES.containsKey(typeExp)) { typeExp = null; } } + + // get the buffering strategy + BufferingStrategies strategy = BufferingController.stringToBufferingStrategies(scope, (String)GamaPreferences.get(GamaPreferences.PREF_SAVE_BUFFERING_STRATEGY).value(scope)); + if (bufferingStrategy != null) { + strategy = BufferingController.stringToBufferingStrategies(scope, (String)bufferingStrategy.value(scope)); + } + try { Files.createDirectories(fileToSave.toPath().getParent()); boolean exists = fileToSave.exists(); final boolean rewrite = shouldOverwrite(scope); - if (rewrite && exists) { - fileToSave.delete(); - exists = false; - } + IExpression header = getFacet(IKeyword.HEADER); final boolean addHeader = !exists && (header == null || Cast.asBool(scope, header.value(scope))); final String type = (typeExp != null ? typeExp : "text").trim().toLowerCase(); @@ -443,7 +485,8 @@ public Object privateExecuteIn(final IScope scope) throws GamaRuntimeException { IType itemType = item.getGamlType(); ISaveDelegate delegate = findDelegate(itemType, type); if (delegate != null) { - delegate.save(scope, item, fileToSave, code, addHeader, type, attributesFacet); + var saveOptions = new SaveOptions(code, addHeader, type, attributesFacet, strategy, rewrite); + delegate.save(scope, item, fileToSave, saveOptions); return Cast.asString(scope, file.value(scope)); } throw GamaRuntimeException.error("Format not recognized: " + type, scope); diff --git a/gama.core/src/gama/gaml/statements/WriteStatement.java b/gama.core/src/gama/gaml/statements/WriteStatement.java index dbbac3ddae..82375b0145 100644 --- a/gama.core/src/gama/gaml/statements/WriteStatement.java +++ b/gama.core/src/gama/gaml/statements/WriteStatement.java @@ -10,9 +10,6 @@ ********************************************************************************************************/ package gama.gaml.statements; -import gama.annotations.precompiler.IConcept; -import gama.annotations.precompiler.IOperatorCategory; -import gama.annotations.precompiler.ISymbolKind; import gama.annotations.precompiler.GamlAnnotations.doc; import gama.annotations.precompiler.GamlAnnotations.example; import gama.annotations.precompiler.GamlAnnotations.facet; @@ -22,15 +19,28 @@ import gama.annotations.precompiler.GamlAnnotations.symbol; import gama.annotations.precompiler.GamlAnnotations.test; import gama.annotations.precompiler.GamlAnnotations.usage; +import gama.annotations.precompiler.IConcept; +import gama.annotations.precompiler.IOperatorCategory; +import gama.annotations.precompiler.ISymbolKind; import gama.core.common.interfaces.IKeyword; +import gama.core.common.preferences.GamaPreferences; import gama.core.common.util.StringUtils; import gama.core.metamodel.agent.IAgent; +import gama.core.runtime.GAMA; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.GamaColor; +import gama.gaml.compilation.IDescriptionValidator; +import gama.gaml.compilation.annotations.validator; import gama.gaml.descriptions.IDescription; +import gama.gaml.descriptions.StatementDescription; import gama.gaml.expressions.IExpression; +import gama.gaml.interfaces.IGamlIssue; import gama.gaml.operators.Cast; +import gama.gaml.operators.Strings; +import gama.gaml.statements.WriteStatement.WriteValidator; import gama.gaml.types.IType; /** @@ -48,24 +58,65 @@ @inside ( kinds = { ISymbolKind.BEHAVIOR, ISymbolKind.SEQUENCE_STATEMENT, ISymbolKind.LAYER }) @facets ( - value = { @facet ( + value = { + @facet ( name = IKeyword.COLOR, type = IType.COLOR, optional = true, doc = @doc ("The color with wich the message will be displayed. Note that different simulations will have different (default) colors to use for this purpose if this facet is not specified")), - @facet ( - name = IKeyword.MESSAGE, - type = IType.NONE, - optional = false, - doc = @doc ("the message to display. Modelers can add some formatting characters to the message (carriage returns, tabs, or Unicode characters), which will be used accordingly in the console.")), }, + @facet ( + name = IKeyword.END, + type = IType.STRING, + optional = true, + doc = @doc ("The string to be appened at the end of the message. By default it's a new line character: '\\n' or '\\r\\n' depending on the operating system." ) + ), + @facet ( + name = IKeyword.BUFFERING, + type = { IType.STRING}, + optional = true, + doc = @doc ( + value = "Allows to specify a buffering strategy to write in the console. Accepted values are `" + BufferingController.PER_CYCLE_BUFFERING +"` and `" + BufferingController.PER_SIMULATION_BUFFERING + "`, `" + BufferingController.NO_BUFFERING + "`. " + + "In the case of `"+ BufferingController.PER_CYCLE_BUFFERING +"` or `"+ BufferingController.PER_SIMULATION_BUFFERING +"`, all the write operations in the simulation which used these values would be " + + "executed all at once at the end of the cycle or simulation while keeping the initial order. In case of '" + BufferingController.PER_AGENT + + "' all operations will be released when the agent is killed (or the simulation ends). Those strategies can be used to optimise a " + + "simulation's execution time on models that extensively write in files. " + + "The `" + BufferingController.NO_BUFFERING + "` (which is the system's default) will directly write into the file.") + ), + @facet ( + name = IKeyword.MESSAGE, + type = IType.NONE, + optional = false, + doc = @doc ("the message to display. Modelers can add some formatting characters to the message (carriage returns, tabs, or Unicode characters), which will be used accordingly in the console.")), }, + omissible = IKeyword.MESSAGE) @doc ( value = "The statement makes the agent output an arbitrary message in the console.", usages = { @usage ( value = "Outputting a message", examples = { @example ("write \"This is a message from \" + self;") }) }) +@validator(WriteValidator.class) public class WriteStatement extends AbstractStatement { + + public static class WriteValidator implements IDescriptionValidator { + + @Override + public void validate(StatementDescription description) { + final StatementDescription desc = description; + final IExpression bufferingStrategy = desc.getFacetExpr(IKeyword.BUFFERING); + + + if (bufferingStrategy != null && ! BufferingController.BUFFERING_STRATEGIES.contains(bufferingStrategy.literalValue())) { + desc.error("The value for buffering must be '" + BufferingController.NO_BUFFERING + "', '" + + BufferingController.PER_CYCLE_BUFFERING + "', '" + + BufferingController.PER_AGENT + "'" + + "' or '" + BufferingController.PER_SIMULATION_BUFFERING +"'.", + IGamlIssue.WRONG_TYPE); + } + } + + } + static { // DEBUG.ON(); } @@ -81,7 +132,11 @@ public String getTrace(final IScope scope) { /** The color. */ final IExpression color; + + final IExpression bufferingStrategy; + final IExpression end; + /** * Instantiates a new write statement. * @@ -92,6 +147,8 @@ public WriteStatement(final IDescription desc) { super(desc); message = getFacet(IKeyword.MESSAGE); color = getFacet(IKeyword.COLOR); + bufferingStrategy = getFacet(IKeyword.BUFFERING); + end = getFacet(IKeyword.END); } @Override @@ -102,10 +159,25 @@ public Object privateExecuteIn(final IScope scope) throws GamaRuntimeException { mes = Cast.asString(scope, message.value(scope)); if (mes == null) { mes = "nil"; } GamaColor rgb = null; - if (color != null) { rgb = (GamaColor) color.value(scope); } + if (color != null) { + rgb = (GamaColor) color.value(scope); + } + BufferingStrategies strategy = BufferingController.stringToBufferingStrategies(scope, (String)GamaPreferences.get(GamaPreferences.PREF_WRITE_BUFFERING_STRATEGY).value(scope)); + if (bufferingStrategy != null) { + strategy = BufferingController.stringToBufferingStrategies(scope, Cast.asString(scope,bufferingStrategy.value(scope))); + } + + var messageToSend = new StringBuilder(mes); + if (end != null) { + messageToSend.append(Cast.asString(scope, end)); + } + else { + messageToSend.append(Strings.LN); + } + // DEBUG.OUT( // "" + getName() + " asking to write and passing " + scope.getRoot() + " as the corresponding agent"); - scope.getGui().getConsole().informConsole(mes, scope.getRoot(), rgb); + GAMA.askWriteConsole(scope, messageToSend, rgb, strategy); } return mes; } diff --git a/gama.core/src/gama/gaml/statements/save/ASCSaver.java b/gama.core/src/gama/gaml/statements/save/ASCSaver.java index d6f84116dc..02e4696773 100644 --- a/gama.core/src/gama/gaml/statements/save/ASCSaver.java +++ b/gama.core/src/gama/gaml/statements/save/ASCSaver.java @@ -15,11 +15,13 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.util.Set; import gama.core.metamodel.topology.grid.GridPopulation; import gama.core.metamodel.topology.projection.ProjectionFactory; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.util.matrix.GamaField; import gama.gaml.expressions.IExpression; import gama.gaml.operators.Cast; @@ -53,24 +55,10 @@ public class ASCSaver extends AbstractSaver { * Signals that an I/O exception has occurred. */ @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) throws IOException { - - if (file.exists()) { file.delete(); } - - FileWriter fileWriter = null; - - try { - fileWriter = new FileWriter(file); + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) throws IOException { + try (FileWriter fileWriter = new FileWriter(file, StandardCharsets.UTF_8, false)){ save(scope, item, fileWriter); }finally { - // cleanup in case of failure in the save - if (fileWriter != null) { - try { - fileWriter.close(); - } finally {} - } - ProjectionFactory.saveTargetCRSAsPRJFile(scope, file.getAbsolutePath()); } } diff --git a/gama.core/src/gama/gaml/statements/save/AbstractShapeSaver.java b/gama.core/src/gama/gaml/statements/save/AbstractShapeSaver.java index e3a3a38a35..d77e849ece 100644 --- a/gama.core/src/gama/gaml/statements/save/AbstractShapeSaver.java +++ b/gama.core/src/gama/gaml/statements/save/AbstractShapeSaver.java @@ -41,6 +41,7 @@ import gama.core.metamodel.topology.projection.IProjection; import gama.core.metamodel.topology.projection.SimpleScalingProjection; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.GamaListFactory; import gama.core.util.GamaMapFactory; @@ -93,9 +94,8 @@ public abstract class AbstractShapeSaver extends AbstractSaver { * the gama runtime exception */ @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) throws GamaRuntimeException { - save(scope, item, file, code, attributesToSave); + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) throws GamaRuntimeException { + save(scope, item, file, saveOptions.code, saveOptions.attributesToSave); } /** diff --git a/gama.core/src/gama/gaml/statements/save/CSVSaver.java b/gama.core/src/gama/gaml/statements/save/CSVSaver.java index bd325e9613..5dd52eeac6 100644 --- a/gama.core/src/gama/gaml/statements/save/CSVSaver.java +++ b/gama.core/src/gama/gaml/statements/save/CSVSaver.java @@ -10,17 +10,15 @@ package gama.gaml.statements.save; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; import java.util.Collection; import java.util.Set; import gama.core.common.util.StringUtils; import gama.core.metamodel.agent.IAgent; +import gama.core.runtime.GAMA; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.GamaListFactory; import gama.core.util.IList; @@ -54,11 +52,11 @@ public class CSVSaver extends AbstractSaver { * @throws GamaRuntimeException * the gama runtime exception */ - public void save(final IScope scope, final IExpression item, final OutputStream os, final boolean header) - throws GamaRuntimeException { - if (os == null) return; - save(scope, new OutputStreamWriter(os), header, item); - } +// public void save(final IScope scope, final IExpression item, final OutputStream os, final boolean header) +// throws GamaRuntimeException { +// if (os == null) return; +// save(scope, new OutputStreamWriter(os), header, item); +// } /** * Save. @@ -77,24 +75,68 @@ public void save(final IScope scope, final IExpression item, final OutputStream * Signals that an I/O exception has occurred. */ @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) throws GamaRuntimeException, IOException { - - FileWriter fileWriter = null; - - try { - fileWriter = new FileWriter(file, true); - save(scope, fileWriter, addHeader, item); - }catch(IOException ex) { - // cleanup in case of failure in the save - if (fileWriter != null) { - try { - fileWriter.close(); - } finally {} + + StringBuilder sb = new StringBuilder(); + final IType itemType = item.getGamlType(); + final SpeciesDescription sd; + if (itemType.isAgentType()) { + sd = itemType.getSpecies(); + } else if (itemType.getContentType().isAgentType()) { + sd = itemType.getContentType().getSpecies(); + } else { + sd = null; + } + final Object value = item.value(scope); + final IList values = + itemType.isContainer() ? Cast.asList(scope, value) : GamaListFactory.create(scope, itemType, value); + if (values.isEmpty()) return; + char del = AbstractCSVManipulator.getDefaultDelimiter(); + if (sd != null) { + final Collection attributeNames = sd.getAttributeNames(); + attributeNames.removeAll(SaveStatement.NON_SAVEABLE_ATTRIBUTE_NAMES); + if (saveOptions.addHeader) { + sb.append("cycle" + del + "name;location.x" + del + "location.y" + del + "location.z"); + for (final String v : attributeNames) { sb.append(del + v); } + sb.append(Strings.LN); } - throw ex; + for (final Object obj : values) { + if (obj instanceof IAgent) { + final IAgent ag = Cast.asAgent(scope, obj); + sb.append(scope.getClock().getCycle() + del + ag.getName().replace(';', ',') + del + + ag.getLocation().getX() + del + ag.getLocation().getY() + del + + ag.getLocation().getZ()); + for (final String v : attributeNames) { + String val = StringUtils.toGaml(ag.getDirectVarValue(scope, v), false).replace(';', ','); + if (val.startsWith("'") && val.endsWith("'") + || val.startsWith("\"") && val.endsWith("\"")) { + val = val.substring(1, val.length() - 1); + } + sb.append(del + val); + } + sb.append(Strings.LN); + } + } + } else { + if (saveOptions.addHeader) { + sb.append(item.serializeToGaml(true).replace("]", "").replace("[", "").replace(',', del)); + sb.append(Strings.LN); + } + if (itemType.id() == IType.MATRIX) { + GamaMatrix matrix = (GamaMatrix) value; + matrix.rowByRow(scope, v -> sb.append(toCleanString(v)), () -> sb.append(del), + () -> sb.append(Strings.LN)); + } else { + final int size = values.size(); + for (int i = 0; i < size; i++) { + if (i > 0) { sb.append(del); } + sb.append(toCleanString(values.get(i))); + } + } + sb.append(Strings.LN); } + GAMA.askWriteFile(scope, file, sb, saveOptions); } /** @@ -111,74 +153,9 @@ public void save(final IScope scope, final IExpression item, final File file, fi * @throws GamaRuntimeException * the gama runtime exception */ - private void save(final IScope scope, final Writer fw, final boolean header, final IExpression item) + private void save(final IScope scope, final File file, final boolean header, final IExpression item, final BufferingStrategies bufferingStrategy) throws GamaRuntimeException { - try (fw) { - final IType itemType = item.getGamlType(); - final SpeciesDescription sd; - if (itemType.isAgentType()) { - sd = itemType.getSpecies(); - } else if (itemType.getContentType().isAgentType()) { - sd = itemType.getContentType().getSpecies(); - } else { - sd = null; - } - final Object value = item.value(scope); - final IList values = - itemType.isContainer() ? Cast.asList(scope, value) : GamaListFactory.create(scope, itemType, value); - if (values.isEmpty()) return; - char del = AbstractCSVManipulator.getDefaultDelimiter(); - if (sd != null) { - final Collection attributeNames = sd.getAttributeNames(); - attributeNames.removeAll(SaveStatement.NON_SAVEABLE_ATTRIBUTE_NAMES); - if (header) { - fw.write("cycle" + del + "name;location.x" + del + "location.y" + del + "location.z"); - for (final String v : attributeNames) { fw.write(del + v); } - fw.write(Strings.LN); - } - for (final Object obj : values) { - if (obj instanceof IAgent) { - final IAgent ag = Cast.asAgent(scope, obj); - fw.write(scope.getClock().getCycle() + del + ag.getName().replace(';', ',') + del - + ag.getLocation().getX() + del + ag.getLocation().getY() + del - + ag.getLocation().getZ()); - for (final String v : attributeNames) { - String val = StringUtils.toGaml(ag.getDirectVarValue(scope, v), false).replace(';', ','); - if (val.startsWith("'") && val.endsWith("'") - || val.startsWith("\"") && val.endsWith("\"")) { - val = val.substring(1, val.length() - 1); - } - fw.write(del + val); - } - fw.write(Strings.LN); - } - - } - } else { - if (header) { - fw.write(item.serializeToGaml(true).replace("]", "").replace("[", "").replace(',', del)); - fw.write(Strings.LN); - } - if (itemType.id() == IType.MATRIX) { - GamaMatrix matrix = (GamaMatrix) value; - matrix.rowByRow(scope, v -> fw.write(toCleanString(v)), () -> fw.write(del), - () -> fw.write(Strings.LN)); - } else { - final int size = values.size(); - for (int i = 0; i < size; i++) { - if (i > 0) { fw.write(del); } - fw.write(toCleanString(values.get(i))); - } - } - fw.write(Strings.LN); - } - - } catch (final GamaRuntimeException e) { - throw e; - } catch (final Exception e) { - throw GamaRuntimeException.create(e, scope); - } - + } /** diff --git a/gama.core/src/gama/gaml/statements/save/GeoTiffSaver.java b/gama.core/src/gama/gaml/statements/save/GeoTiffSaver.java index ccba0017bb..aeaf0a7b33 100644 --- a/gama.core/src/gama/gaml/statements/save/GeoTiffSaver.java +++ b/gama.core/src/gama/gaml/statements/save/GeoTiffSaver.java @@ -26,6 +26,7 @@ import gama.core.metamodel.topology.projection.IProjection; import gama.core.metamodel.topology.projection.ProjectionFactory; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.util.matrix.GamaField; import gama.gaml.expressions.IExpression; import gama.gaml.operators.Cast; @@ -60,11 +61,14 @@ public class GeoTiffSaver extends AbstractSaver { * Signals that an I/O exception has occurred. */ @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) throws IOException { + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) throws IOException { if (file == null) return; File f = file; - if (f.exists()) { f.delete(); } + // in case it already exists we delete it, if deletion fail we cancel the saving + if (f.exists() && !f.delete()) { + return; + } + try { Object v = item.value(scope); if (v instanceof GamaField gf) { diff --git a/gama.core/src/gama/gaml/statements/save/GraphSaver.java b/gama.core/src/gama/gaml/statements/save/GraphSaver.java index d73306ffe1..5da16d097e 100644 --- a/gama.core/src/gama/gaml/statements/save/GraphSaver.java +++ b/gama.core/src/gama/gaml/statements/save/GraphSaver.java @@ -17,6 +17,7 @@ import org.jgrapht.nio.GraphExporter; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.graph.writer.GraphExporters; import gama.gaml.expressions.IExpression; @@ -43,12 +44,11 @@ public class GraphSaver extends AbstractSaver { */ @Override @SuppressWarnings ("unchecked") - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) { - GraphExporter exp = GraphExporters.getGraphWriter(type); + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) { + GraphExporter exp = GraphExporters.getGraphWriter(saveOptions.type); final var g = Cast.asGraph(scope, item); if (g != null) { - if (exp == null) throw GamaRuntimeException.error("Format is not recognized ('" + type + "')", scope); + if (exp == null) throw GamaRuntimeException.error("Format is not recognized ('" + saveOptions.type + "')", scope); exp.exportGraph(g, file.getAbsoluteFile()); } } diff --git a/gama.core/src/gama/gaml/statements/save/JsonSaver.java b/gama.core/src/gama/gaml/statements/save/JsonSaver.java index c69658a4ad..ea974a344a 100644 --- a/gama.core/src/gama/gaml/statements/save/JsonSaver.java +++ b/gama.core/src/gama/gaml/statements/save/JsonSaver.java @@ -13,9 +13,11 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.util.Set; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.core.runtime.exceptions.GamaRuntimeException; import gama.core.util.file.json.Json; import gama.core.util.file.json.WriterConfig; @@ -30,10 +32,9 @@ public class JsonSaver extends AbstractSaver { @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions saveOptions) throws GamaRuntimeException { - try (Writer fw = new FileWriter(file, true)) { + try (Writer fw = new FileWriter(file, StandardCharsets.UTF_8, !saveOptions.rewrite)) { Json.getNew().valueOf(item.value(scope)).writeTo(fw, WriterConfig.PRETTY_PRINT); } catch (final GamaRuntimeException e) { throw e; diff --git a/gama.core/src/gama/gaml/statements/save/KmlSaver.java b/gama.core/src/gama/gaml/statements/save/KmlSaver.java index 81c131a641..1a42c3a7c6 100644 --- a/gama.core/src/gama/gaml/statements/save/KmlSaver.java +++ b/gama.core/src/gama/gaml/statements/save/KmlSaver.java @@ -13,6 +13,7 @@ import java.util.Set; import gama.core.runtime.IScope; +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; import gama.gaml.expressions.IExpression; import gama.gaml.types.GamaKmlExport; @@ -34,12 +35,11 @@ public class KmlSaver extends AbstractSaver { * the type */ @Override - public void save(final IScope scope, final IExpression item, final File file, final String code, - final boolean addHeader, final String type, final Object attributesToSave) { + public void save(final IScope scope, final IExpression item, final File file, final SaveOptions options) { final Object kml = item.value(scope); String path = file.getAbsolutePath(); if (!(kml instanceof GamaKmlExport export)) return; - if ("kml".equals(type)) { + if ("kml".equals(options.type)) { export.saveAsKml(scope, path); } else { export.saveAsKmz(scope, path); diff --git a/gama.core/src/gama/gaml/statements/save/SaveOptions.java b/gama.core/src/gama/gaml/statements/save/SaveOptions.java new file mode 100644 index 0000000000..10eae7b224 --- /dev/null +++ b/gama.core/src/gama/gaml/statements/save/SaveOptions.java @@ -0,0 +1,37 @@ +package gama.gaml.statements.save; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import gama.core.runtime.concurrent.BufferingController.BufferingStrategies; + +public class SaveOptions { + + public final String code; + public final boolean addHeader; + public final String type; + public final Object attributesToSave; + public final BufferingStrategies bufferingStrategy; + public final boolean rewrite; + protected Charset writeCharset; + + public SaveOptions(final String code, final boolean addHeader, final String type, final Object attributesToSave, + BufferingStrategies bufferingStrategy, final boolean rewrite) { + this.code = code; + this.addHeader = addHeader; + this.type = type; + this.attributesToSave = attributesToSave; + this.bufferingStrategy = bufferingStrategy; + this.rewrite = rewrite; + writeCharset = StandardCharsets.UTF_8; + } + + public void setCharSet(Charset c) { + writeCharset = c; + } + + public Charset getCharset() { + return writeCharset; + } + +} diff --git a/gama.core/src/gama/gaml/statements/save/ShapeSaver.java b/gama.core/src/gama/gaml/statements/save/ShapeSaver.java index 141f7cb455..0bd3e06487 100644 --- a/gama.core/src/gama/gaml/statements/save/ShapeSaver.java +++ b/gama.core/src/gama/gaml/statements/save/ShapeSaver.java @@ -60,7 +60,7 @@ public void internalSave(final IScope scope, final File f, final List