From 77df594c2bfaf7e05da30d3af99eff7efd78218b Mon Sep 17 00:00:00 2001 From: Pyrix25633 Date: Tue, 22 Nov 2022 16:57:33 +0100 Subject: [PATCH] 1.6.0: Major improvements with async scan and Dictionary --- Files.cs | 68 ++++++++------------------ Logger.cs | 95 +++++++++++++++++++++--------------- Makefile | 3 +- Program.cs | 140 ++++++++++++++++++++++++++++++----------------------- 4 files changed, 160 insertions(+), 146 deletions(-) diff --git a/Files.cs b/Files.cs index a738999..88fd7de 100644 --- a/Files.cs +++ b/Files.cs @@ -2,30 +2,30 @@ public class DirectoryEntry { /// /// Initializer /// - public DirectoryEntry(string path, string from) { + public DirectoryEntry(string path, string relativePath) { fileInfo = new FileInfo(path); - relativePath = path.Substring(from.Length + 1); + this.relativePath = relativePath; } public string relativePath; public FileInfo fileInfo; public Reason reason; /// /// Function to search in the destination folder for the same file and compare them - /// (, , ) + /// (, , ) /// - /// The list of all files in the destination folder + /// The dictionaty of all files in the destination folder /// If all extensions have to be checked for content differencies /// The list of the extensions to check for content differencies /// Returns true if the file has to be copied - public bool ToCopy(ref DirectoryEntry[] destinationList, bool allExtensions, string[] extensions) { - Int32 pos; - bool found = IsInList(destinationList, out pos); - if(!found) { + public bool ToCopy(ref Dictionary destinationDictionary, bool allExtensions, string[] extensions) { + DirectoryEntry e; + try {e = destinationDictionary[relativePath];} + catch(Exception) { + // Does not exist reason = Reason.CopyNotThere; return true; } - DirectoryEntry e = destinationList[pos]; - destinationList = RemoveAt(destinationList, pos); + // Skip if directory if((e.fileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) return false; // Check size if(fileInfo.Length != e.fileInfo.Length) { @@ -57,46 +57,20 @@ public bool ToCopy(ref DirectoryEntry[] destinationList, bool allExtensions, str } /// /// Function to know of a file has to be removed - /// () + /// () /// - /// The list of files in the source folder + /// The dictionary of files in the source folder /// Returns true if the file has to be removed - public bool ToRemove(DirectoryEntry[] sourceList) { - bool toRemove = !IsInList(sourceList, out _); - if(toRemove) reason = Reason.Remove; - return toRemove; - } - /// - /// Function to search a file in a list of files - /// (, ) - /// - /// The list of files - /// The returned position - /// Returns true if the file is in the list - private bool IsInList(DirectoryEntry[] list, out Int32 pos) { - Int32 length = list.Length; - for(pos = 0; pos < length; pos++) { - if(relativePath == list[pos].relativePath) return true; + public bool ToRemove(ref Dictionary sourceDictionary) { + DirectoryEntry e; + try { + e = sourceDictionary[relativePath]; + return false; + } + catch(Exception) { + reason = Reason.Remove; + return true; } - return false; - } - /// - /// Function to search remove an item from a DirectoryEntry array - /// (, ) - /// - /// The source DirectoryEntry array - /// The index of the item to remove - /// Returns the source array without the item to remove - public static DirectoryEntry[] RemoveAt(DirectoryEntry[] source, Int32 index) { - Int32 lenght = source.Length; - DirectoryEntry[] dest = new DirectoryEntry[lenght - 1]; - if( index > 0 ) - Array.Copy(source, 0, dest, 0, index); - - if( index < lenght - 1 ) - Array.Copy(source, index + 1, dest, index, lenght - index - 1); - - return dest; } } diff --git a/Logger.cs b/Logger.cs index b455005..a45da66 100644 --- a/Logger.cs +++ b/Logger.cs @@ -4,7 +4,8 @@ public class Logger { private static string barFull = "█", barEmpty = " "; private static string? logfilename; private static StreamWriter? logstream; - private static Task? logTask; + private static List logTasks = new List(); + private static int ticket = 0; /// /// Function to initialize the file logging /// () @@ -30,8 +31,9 @@ public static void ReinitializeLogging() { /// public static async void TerminateLogging() { if(logstream != null) { - if(logTask != null) await logTask; + if(logTasks.Count != 0) await logTasks[logTasks.Count - 1]; logstream.Close(); + logTasks = new List(); } } /// @@ -39,62 +41,79 @@ public static async void TerminateLogging() { /// () /// /// The message to output - public static void Success(string message) { + public static async void Success(string message) { + int t = ticket++; string time = TimeString(); - Console.ForegroundColor = ConsoleColor.DarkGray; - Console.Write(time); - Console.ForegroundColor = ConsoleColor.Green; - Console.Write("(Success) "); - Console.ForegroundColor = ConsoleColor.White; - Console.WriteLine(message); - Console.ResetColor(); - if(logstream != null) logTask = logstream.WriteLineAsync(time + "(Success) " + message); + await logTasks.ElementAt(t); + logTasks.Append(Task.Run(() => { + Console.ForegroundColor = ConsoleColor.DarkGray; + Console.Write(time); + Console.ForegroundColor = ConsoleColor.Green; + Console.Write("(Success) "); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(message); + Console.ResetColor(); + if(logstream != null) logstream.WriteLineAsync(time + "(Success) " + message); + })); } /// /// Function to output an info message /// () /// /// The message to output - public static void Info(string message) { - Console.ForegroundColor = ConsoleColor.DarkGray; - Console.Write(TimeString()); - Console.ForegroundColor = ConsoleColor.Cyan; - Console.Write("(Info) "); - Console.ForegroundColor = ConsoleColor.White; - Console.WriteLine(message); - Console.ResetColor(); + public static async void Info(string message) { + int t = ticket++; + string time = TimeString(); + await logTasks.ElementAt(t); + logTasks.Append(Task.Run(() => { + Console.ForegroundColor = ConsoleColor.DarkGray; + Console.Write(time); + Console.ForegroundColor = ConsoleColor.Cyan; + Console.Write("(Info) "); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(message); + Console.ResetColor(); + })); } /// /// Function to output a warning message /// () /// /// The message to output - public static void Warning(string message) { - string time = TimeString(); - Console.ForegroundColor = ConsoleColor.DarkGray; - Console.Write(time); - Console.ForegroundColor = ConsoleColor.Yellow; - Console.Write("(Warning) "); - Console.ForegroundColor = ConsoleColor.White; - Console.WriteLine(message); - Console.ResetColor(); - if(logstream != null) logTask = logstream.WriteLineAsync(time + "(Warning) " + message); + public static async void Warning(string message) { + int t = ticket++; + string time = TimeString(); + await logTasks.ElementAt(t); + logTasks.Append(Task.Run(() => { + Console.ForegroundColor = ConsoleColor.DarkGray; + Console.Write(time); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.Write("(Warning) "); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(message); + Console.ResetColor(); + if(logstream != null) logstream.WriteLineAsync(time + "(Warning) " + message); + })); } /// /// Function to output an error message /// () /// /// The message to output - public static void Error(string message) { + public static async void Error(string message) { + int t = ticket++; string time = TimeString(); - Console.ForegroundColor = ConsoleColor.DarkGray; - Console.Write(time); - Console.ForegroundColor = ConsoleColor.Red; - Console.Write("(Error) "); - Console.ForegroundColor = ConsoleColor.White; - Console.WriteLine(message); - Console.ResetColor(); - if(logstream != null) logTask = logstream.WriteLineAsync(time + "(Error) " + message); + await logTasks.ElementAt(t); + logTasks.Append(Task.Run(() => { + Console.ForegroundColor = ConsoleColor.DarkGray; + Console.Write(time); + Console.ForegroundColor = ConsoleColor.Red; + Console.Write("(Error) "); + Console.ForegroundColor = ConsoleColor.White; + Console.WriteLine(message); + Console.ResetColor(); + if(logstream != null) logstream.WriteLineAsync(time + "(Error) " + message); + })); } /// /// Function to clear the last console line diff --git a/Makefile b/Makefile index 6e87dd6..4c30448 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,5 @@ run-dotnet-debug: > dotnet ./transfer/docker/debug/backup-utility.dll -- -s test/source -d test/destination -r test/removed -t 100 -e extensions.txt -l -b run-dotnet-release: -> ./transfer/docker/release/linux-x64/backup-utility \ No newline at end of file +> dotnet publish -c debug +> ./transfer/docker/release/linux-x64/backup-utility -s test/source -d test/destination -r test/removed -l \ No newline at end of file diff --git a/Program.cs b/Program.cs index f48d862..223d9e8 100644 --- a/Program.cs +++ b/Program.cs @@ -2,14 +2,16 @@ using System.IO.Compression; public class Program { - static void Main(string[] args) { + private static Arguments arguments = new Arguments(); + private static EnumerationOptions enumOptions = new EnumerationOptions(); + static async Task Main(string[] args) { // Version string version = "1.5.6"; // Lists string[] sourceList = new string[0], destinationList = new string[0], extensionList = new string[0]; - DirectoryEntry[] sourceInfoList = new DirectoryEntry[0], destinationInfoList = new DirectoryEntry[0], - toCopyList = new DirectoryEntry[0], toRemoveFileList = new DirectoryEntry[0], toRemoveFolderList = new DirectoryEntry[0]; - EnumerationOptions enumOptions = new EnumerationOptions(); + Dictionary sourceInfoDictionary = new Dictionary(); + Dictionary destinationInfoDictionary = new Dictionary(); + DirectoryEntry[] toCopyList = new DirectoryEntry[0], toRemoveFileList = new DirectoryEntry[0], toRemoveFolderList = new DirectoryEntry[0]; enumOptions.RecurseSubdirectories = true; enumOptions.AttributesToSkip = default; // Other variables Int32 length, filesToCopy, filesCopied, foldersToCopy, foldersCopied, @@ -18,7 +20,6 @@ static void Main(string[] args) { Int64 timestamp; string backupFolder = ""; // Parsing arguments - Arguments arguments = new Arguments(); try { arguments.Parse(args); if(arguments.help) return; @@ -107,7 +108,7 @@ static void Main(string[] args) { else { Logger.Info("Extension list not set, only file size will be used to compare files"); } - //Compressed Backup + // Compressed Backup if(arguments.backup) { Logger.Info("Compressed backup: yes"); backupFolder = arguments.destination + "-backups"; @@ -129,62 +130,30 @@ static void Main(string[] args) { // Timestamp timestamp = new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds() + arguments.time; // Scan folders - Logger.Info("Starting source folder scan..."); - try { - sourceList = Directory.EnumerateFileSystemEntries(arguments.source, "*", enumOptions).ToArray(); - Logger.Success("Source folder scanned: " + sourceList.Length + " items found"); - } - catch(Exception e) { - Logger.Error("Error while scanning source folder: " + e); - continue; - } - Logger.Info("Starting destination folder scan..."); - try { - destinationList = Directory.EnumerateFileSystemEntries(arguments.destination, "*", enumOptions).ToArray(); - Logger.Success("Destination folder scanned: " + destinationList.Length + " items found"); - } - catch(Exception e) { - Logger.Error("Error while scanning destination folder: " + e); - continue; - } + Task sourceTask = scanFolder(arguments.source, true); + Task destinationTask = scanFolder(arguments.destination, false); + sourceList = await sourceTask; + destinationList = await destinationTask; // Build file info - Logger.Info("Building source file info list..."); - try { - foreach(string item in sourceList) { - sourceInfoList = sourceInfoList.Append(new DirectoryEntry(item, arguments.source)).ToArray(); - } - Logger.Success("Source file info list built"); - } - catch(Exception e) { - Logger.Error("Error while building source file info list: " + e); - continue; - } + Task> sourceDictionaryTask = buildInfoDictionary(sourceList, arguments.source, true); + Task> destinationDictionaryTask = buildInfoDictionary(sourceList, arguments.destination, false); + sourceInfoDictionary = await sourceDictionaryTask; + destinationInfoDictionary = await sourceDictionaryTask; sourceList = new string[0]; - Logger.Info("Building destination file info list..."); - try { - foreach(string item in destinationList) { - destinationInfoList = destinationInfoList.Append(new DirectoryEntry(item, arguments.destination)).ToArray(); - } - Logger.Success("Destination file info list built"); - } - catch(Exception e) { - Logger.Error("Error while building destination file info list: " + e); - continue; - } destinationList = new string[0]; // Items to copy Logger.Info("Determining items to copy..."); - length = sourceInfoList.Length; filesToCopy = 0; foldersToCopy = 0; sizeToCopy = 0; - for(Int32 i = 0; i < length; i++) { - if(sourceInfoList[i].ToCopy(ref destinationInfoList, arguments.allExtensions, extensionList)) { - toCopyList = toCopyList.Append(sourceInfoList[i]).ToArray(); - if((sourceInfoList[i].fileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) { + foreach(KeyValuePair entry in sourceInfoDictionary) { + DirectoryEntry value = entry.Value; + if(value.ToCopy(ref destinationInfoDictionary, arguments.allExtensions, extensionList)) { + toCopyList = toCopyList.Append(value).ToArray(); + if((value.fileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) { foldersToCopy++; } else { filesToCopy++; - sizeToCopy += (UInt64)sourceInfoList[i].fileInfo.Length; + sizeToCopy += (UInt64)value.fileInfo.Length; } } } @@ -193,18 +162,18 @@ static void Main(string[] args) { Logger.HumanReadableSize(sizeToCopy) + ")"); // Items to remove Logger.Info("Determining items to remove..."); - length = destinationInfoList.Length; filesToRemove = 0; foldersToRemove = 0; sizeToRemove = 0; - for(Int32 i = 0; i < length; i++) { - if(destinationInfoList[i].ToRemove(sourceInfoList)) { - if((destinationInfoList[i].fileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) { - toRemoveFolderList = toRemoveFolderList.Append(destinationInfoList[i]).ToArray(); + foreach(KeyValuePair entry in destinationInfoDictionary) { + DirectoryEntry value = entry.Value; + if(value.ToRemove(ref sourceInfoDictionary)) { + if((value.fileInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) { + toRemoveFolderList = toRemoveFolderList.Append(value).ToArray(); foldersToRemove++; } else { - toRemoveFileList = toRemoveFileList.Append(destinationInfoList[i]).ToArray(); + toRemoveFileList = toRemoveFileList.Append(value).ToArray(); filesToRemove++; - sizeToRemove += (UInt64)destinationInfoList[i].fileInfo.Length; + sizeToRemove += (UInt64)value.fileInfo.Length; } } } @@ -212,7 +181,8 @@ static void Main(string[] args) { filesToRemove.ToString() + " file" + (filesToRemove == 1 ? "" : "s") + " to remove (" + Logger.HumanReadableSize(sizeToRemove) + ")"); // Clear info lists - sourceInfoList = new DirectoryEntry[0]; destinationInfoList = new DirectoryEntry[0]; + sourceInfoDictionary = new Dictionary(); + destinationInfoDictionary = new Dictionary(); // Copy files length = toCopyList.Length; filesCopied = 0; foldersCopied = 0; sizeCopied = 0; @@ -364,4 +334,54 @@ static void Main(string[] args) { Logger.ReinitializeLogging(); } } + + /// + /// Function to get the list of files in a folder + /// (, ) + /// + /// The path + /// True if source folder, false if destination folder + /// Returns the task of a string array + public static async Task scanFolder(string path, bool type) { + return await Task.Run(() => { + string[] array; + Logger.Info("Starting " + (type ? "source" : "destination") + " folder scan..."); + try { + array = Directory.EnumerateFileSystemEntries(arguments.source, "*", enumOptions).ToArray(); + Logger.Success((type ? "Source" : "Destination") + " folder scanned: " + array.Length + " items found"); + } + catch(Exception e) { + Logger.Error("Error while scanning " + (type ? "source" : "destination") + " folder: " + e); + Environment.Exit(1); + array = new string[0]; + } + return array; + }); + } + + /// + /// Function to get the list of files in a folder + /// (, ) + /// + /// The path + /// True if source folder, false if destination folder + /// Returns the task of a string array + public static async Task> buildInfoDictionary(string[] list, string path, bool type) { + return await Task.Run>(() => { + Dictionary dictionary = new Dictionary(); + Logger.Info("Building " + (type ? "source" : "destination") + " file info list..."); + try { + foreach(string item in list) { + string relativePath = item.Substring(path.Length + 1); + dictionary[relativePath] = new DirectoryEntry(item, relativePath); + } + Logger.Success((type ? "Source" : "Destination") + " file info list built"); + } + catch(Exception e) { + Logger.Error("Error while building " + (type ? "source" : "destination") + " file info list: " + e); + Environment.Exit(2); + } + return dictionary; + }); + } } \ No newline at end of file