From a73ad0c509ad5b775fc388bc4135e6a42cc65325 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sat, 8 Jun 2024 21:07:38 +0500 Subject: [PATCH] make the UI more responsive on disc scan and fs analysis stages --- IrdLibraryClient/IrdFormat/IsoHeaderParser.cs | 21 +++++++-- Ps3DiscDumper/Decrypter.cs | 2 +- Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs | 10 ++-- Ps3DiscDumper/Dumper.cs | 47 ++++++++++++------- Ps3DiscDumper/Utils/StreamEx.cs | 4 +- Tests/IrdTests.cs | 2 +- UI.Avalonia/UI.Avalonia.csproj | 4 +- UI.Avalonia/ViewModels/MainViewModel.cs | 41 +++++++++++++--- 8 files changed, 93 insertions(+), 38 deletions(-) diff --git a/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs b/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs index 2aebc6d..943518f 100644 --- a/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs +++ b/IrdLibraryClient/IrdFormat/IsoHeaderParser.cs @@ -5,12 +5,14 @@ namespace IrdLibraryClient.IrdFormat; public static class IsoHeaderParser { - public static (List files, List dirs) GetFilesystemStructure(this CDReader reader) + public static async Task<(List files, List dirs)> GetFilesystemStructureAsync(this CDReader reader, CancellationToken cancellationToken) { + Log.Debug("Scanning filesystem structure…"); var fsObjects = reader.GetFileSystemEntries(reader.Root.FullName).ToList(); var nextLevel = new List(); - var filePaths = new List(); + var filePaths = new List(20_000); var dirPaths = new List(); + var cnt = 0; while (fsObjects.Any()) { foreach (var path in fsObjects) @@ -24,6 +26,11 @@ public static (List files, List dirs) GetFilesystemStruct } else Log.Warn($"Unknown filesystem object: {path}"); + if (++cnt <= 200) + continue; + + await Task.Yield(); + cnt = 0; } (fsObjects, nextLevel) = (nextLevel, fsObjects); nextLevel.Clear(); @@ -38,11 +45,12 @@ public static (List files, List dirs) GetFilesystemStruct .Select(di => new DirRecord(di.dir, new(di.info.CreationTimeUtc, di.info.LastWriteTimeUtc))) .ToList(); + Log.Debug("Building file cluster map…"); var fileList = new List(); foreach (var filename in filenames) { var targetFilename = filename.TrimStart('\\'); - if (targetFilename.EndsWith(".")) + if (targetFilename.EndsWith('.')) { Log.Warn($"Fixing potential mastering error in {filename}"); targetFilename = targetFilename.TrimEnd('.'); @@ -59,10 +67,13 @@ public static (List files, List dirs) GetFilesystemStruct var parent = fileInfo.Parent; var parentInfo = new DirRecord(parent.FullName.TrimStart('\\').Replace('\\', Path.DirectorySeparatorChar), new(parent.CreationTimeUtc, parent.LastWriteTimeUtc)); fileList.Add(new(targetFilename, startSector, lengthInSectors, length, recordInfo, parentInfo)); + if (++cnt <= 200) + continue; + + await Task.Yield(); + cnt = 0; } fileList = fileList.OrderBy(r => r.StartSector).ToList(); - - return (files: fileList, dirs: dirList); } diff --git a/Ps3DiscDumper/Decrypter.cs b/Ps3DiscDumper/Decrypter.cs index 870fad3..e232469 100644 --- a/Ps3DiscDumper/Decrypter.cs +++ b/Ps3DiscDumper/Decrypter.cs @@ -143,7 +143,7 @@ public override int Read(byte[] buffer, int offset, int count) Log.Debug($"Block has only {(readCount % 16) * 8} bits of data, reading raw sector..."); discStream.Seek(SectorPosition * sectorSize, SeekOrigin.Begin); var newTmpSector = new byte[sectorSize]; - discStream.ReadExact(newTmpSector, 0, sectorSize); + discStream.ReadExactly(newTmpSector, 0, sectorSize); if (!newTmpSector.Take(readCount).SequenceEqual(tmpSector.Take(readCount))) Log.Warn($"Filesystem data and raw data do not match for sector 0x{SectorPosition:x8}"); tmpSector = newTmpSector; diff --git a/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs b/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs index 4960119..c2d82ec 100644 --- a/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs +++ b/Ps3DiscDumper/DiscInfo/DiscInfoConverter.cs @@ -2,6 +2,8 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using DiscUtils.Iso9660; using IrdLibraryClient; using IrdLibraryClient.IrdFormat; @@ -11,18 +13,18 @@ namespace Ps3DiscDumper.DiscInfo; public static class DiscInfoConverter { - public static DiscInfo ToDiscInfo(this Ird ird) + public static async Task ToDiscInfoAsync(this Ird ird, CancellationToken cancellationToken) { List fsInfo; var sectorSize = 2048L; using (var stream = new MemoryStream()) { using (var headerStream = new MemoryStream(ird.Header)) - using (var gzipStream = new GZipStream(headerStream, CompressionMode.Decompress)) - gzipStream.CopyTo(stream); + await using (var gzipStream = new GZipStream(headerStream, CompressionMode.Decompress)) + await gzipStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false); stream.Seek(0, SeekOrigin.Begin); var reader = new CDReader(stream, true, true); - (fsInfo, _) = reader.GetFilesystemStructure(); + (fsInfo, _) = await reader.GetFilesystemStructureAsync(cancellationToken).ConfigureAwait(false); sectorSize = reader.ClusterSize; } var checksums = new Dictionary>(ird.FileCount); diff --git a/Ps3DiscDumper/Dumper.cs b/Ps3DiscDumper/Dumper.cs index fd3bdf8..f233dd2 100644 --- a/Ps3DiscDumper/Dumper.cs +++ b/Ps3DiscDumper/Dumper.cs @@ -378,6 +378,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) Log.Trace($"Getting keys from {keyProvider.GetType().Name}..."); var newKeys = await keyProvider.EnumerateAsync(discKeyCachePath, ProductCode, Cts.Token).ConfigureAwait(false); Log.Trace($"Got {newKeys.Count} keys from {keyProvider.GetType().Name}"); + await Task.Yield(); lock (AllKnownDiscKeys) { foreach (var keyInfo in newKeys) @@ -427,6 +428,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) throw; } Log.Debug($"Found {physicalDrives.Count} physical drives"); + await Task.Yield(); if (physicalDrives.Count == 0) throw new InvalidOperationException("No optical drives were found"); @@ -453,7 +455,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) var sector = tmpDiscReader.PathToClusters(discSfbInfo.FullName).First().Offset; Log.Trace($"PS3_DISC.SFB sector number is {sector}, reading content..."); discStream.Seek(sector * tmpDiscReader.ClusterSize, SeekOrigin.Begin); - discStream.ReadExact(buf, 0, buf.Length); + await discStream.ReadExactlyAsync(buf, 0, buf.Length).ConfigureAwait(false); if (buf.SequenceEqual(discSfbData)) { SelectedPhysicalDevice = drive; @@ -467,6 +469,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) { Log.Debug($"Skipping drive {drive}: {e.Message}"); } + await Task.Yield(); } if (SelectedPhysicalDevice == null) throw new AccessViolationException("Direct disk access to the drive was denied"); @@ -498,6 +501,7 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) Log.Debug($"Using {path} for disc key detection"); break; } + await Task.Yield(); } catch (Exception e) { @@ -517,11 +521,12 @@ public async Task FindDiscKeyAsync(string discKeyCachePath) detectionBytesExpected = expectedBytes; sectorIV = Decrypter.GetSectorIV(detectionRecord.StartSector); Log.Debug($"Initialized {nameof(sectorIV)} ({sectorIV?.Length * 8} bit) for sector {detectionRecord.StartSector}: {sectorIV?.ToHexString()}"); - driveStream.ReadExact(detectionSector, 0, detectionSector.Length); + await driveStream.ReadExactlyAsync(detectionSector, 0, detectionSector.Length).ConfigureAwait(false); string discKey = null; try { var validKeys = untestedKeys.AsParallel().Where(k => !Cts.IsCancellationRequested && IsValidDiscKey(k)).Distinct().ToList(); + await Task.Yield(); if (validKeys.Count > 1) { Log.Warn($"Expected only one valid decryption key, but found {validKeys.Count}:"); @@ -566,29 +571,30 @@ public async Task DumpAsync(string output) while (!string.IsNullOrEmpty(dumpPath) && !Directory.Exists(dumpPath)) { var parent = Path.GetDirectoryName(dumpPath); - if (parent == null || parent == dumpPath) + if (parent is null || parent == dumpPath) dumpPath = null; else dumpPath = parent; } if (filesystemStructure is null) { - (filesystemStructure, emptyDirStructure) = GetFilesystemStructure(); + (filesystemStructure, emptyDirStructure) = await GetFilesystemStructureAsync(Cts.Token).ConfigureAwait(false); var filterDirList = SettingsProvider.Settings.FilterDirList; var prefixList = filterDirList.Select(f => f + Path.DirectorySeparatorChar).ToArray(); if (SettingsProvider.Settings.FilterRequired) { filesystemStructure = filesystemStructure - .Where(r => !filterDirList.Any(f => r.TargetFileName == f) && - !prefixList.Any(p => r.TargetFileName.StartsWith(p))) - .ToList(); + .Where(r => !filterDirList.Any(f => r.TargetFileName == f) + && !prefixList.Any(p => r.TargetFileName.StartsWith(p)) + ).ToList(); emptyDirStructure = emptyDirStructure - .Where(r => !filterDirList.Any(f => r.TargetDirName == f) && - !prefixList.Any(p => r.TargetDirName.StartsWith(p))) - .ToList(); + .Where(r => !filterDirList.Any(f => r.TargetDirName == f) + && !prefixList.Any(p => r.TargetDirName.StartsWith(p)) + ).ToList(); } } - var validators = GetValidationInfo(); + await Task.Yield(); + var validators = await GetValidationInfoAsync().ConfigureAwait(false); if (!string.IsNullOrEmpty(dumpPath)) { var fullOutputPath = Path.GetFullPath(output); @@ -606,11 +612,14 @@ public async Task DumpAsync(string output) Log.Warn($"Target drive might require {diff.AsStorageUnit()} of additional free space"); } } + await Task.Yield(); foreach (var dir in emptyDirStructure) Log.Trace($"Empty dir: {dir}"); + await Task.Yield(); foreach (var file in filesystemStructure) Log.Trace($"0x{file.StartSector:x8}: {file.TargetFileName} ({file.SizeInBytes}, {file.FileInfo.CreationTimeUtc:u})"); + await Task.Yield(); var outputPathBase = Path.Combine(output, OutputDir); Log.Debug($"Output path: {outputPathBase}"); if (!Directory.Exists(outputPathBase)) @@ -648,6 +657,7 @@ public async Task DumpAsync(string output) BrokenFiles.Add((dir.TargetDirName, "Unexpected error: " + ex.Message)); } } + await Task.Yield(); foreach (var file in filesystemStructure) { @@ -733,6 +743,7 @@ select v.Files[file.TargetFileName].Hashes ValidationStatus = false; } } + await Task.Yield(); } while (error && tries > 0 && !Cts.IsCancellationRequested); _ = new FileInfo(outputFilename) @@ -820,36 +831,36 @@ select v.Files[file.TargetFileName].Hashes } - private (List files, List dirs) GetFilesystemStructure() + private async Task<(List files, List dirs)> GetFilesystemStructureAsync(CancellationToken cancellationToken) { var pos = driveStream.Position; var buf = new byte[64 * 1024 * 1024]; driveStream.Seek(0, SeekOrigin.Begin); - driveStream.ReadExact(buf, 0, buf.Length); + await driveStream.ReadExactlyAsync(buf, 0, buf.Length, cancellationToken).ConfigureAwait(false); driveStream.Seek(pos, SeekOrigin.Begin); try { using var memStream = new MemoryStream(buf, false); var reader = new CDReader(memStream, true, true); - return reader.GetFilesystemStructure(); + return await reader.GetFilesystemStructureAsync(cancellationToken).ConfigureAwait(false); } catch (Exception e) { Log.Error(e, "Failed to buffer TOC"); } - return discReader.GetFilesystemStructure(); + return await discReader.GetFilesystemStructureAsync(cancellationToken).ConfigureAwait(false); } - private List GetValidationInfo() + private async Task> GetValidationInfoAsync() { var discInfoList = new List(); foreach (var discKeyInfo in allMatchingKeys.Where(ki => ki.KeyType == KeyType.Ird)) { - var ird = IrdParser.Parse(File.ReadAllBytes(discKeyInfo.FullPath)); + var ird = IrdParser.Parse(await File.ReadAllBytesAsync(discKeyInfo.FullPath).ConfigureAwait(false)); if (!DiscVersion.Equals(ird.GameVersion)) continue; - discInfoList.Add(ird.ToDiscInfo()); + discInfoList.Add(await ird.ToDiscInfoAsync(Cts.Token).ConfigureAwait(false)); } return discInfoList; } diff --git a/Ps3DiscDumper/Utils/StreamEx.cs b/Ps3DiscDumper/Utils/StreamEx.cs index ec1b8d6..ec27d70 100644 --- a/Ps3DiscDumper/Utils/StreamEx.cs +++ b/Ps3DiscDumper/Utils/StreamEx.cs @@ -1,9 +1,11 @@ -using System.IO; +using System; +using System.IO; namespace Ps3DiscDumper.Utils; public static class StreamEx { + [Obsolete] public static int ReadExact(this Stream input, byte[] buffer, int offset, int count) { var result = 0; diff --git a/Tests/IrdTests.cs b/Tests/IrdTests.cs index db4e69e..626f448 100644 --- a/Tests/IrdTests.cs +++ b/Tests/IrdTests.cs @@ -33,7 +33,7 @@ public async Task FileStructureParseTest(string productCode, int expectedFileCou await using var decompressedStream = GetDecompressHeader(ird); var reader = new CDReader(decompressedStream, true, true); - var (files, _) = reader.GetFilesystemStructure(); + var (files, _) = await reader.GetFilesystemStructureAsync(CancellationToken.None).ConfigureAwait(false); Assert.That(files, Has.Count.EqualTo(expectedFileCount)); } diff --git a/UI.Avalonia/UI.Avalonia.csproj b/UI.Avalonia/UI.Avalonia.csproj index 8d29048..164c631 100644 --- a/UI.Avalonia/UI.Avalonia.csproj +++ b/UI.Avalonia/UI.Avalonia.csproj @@ -11,8 +11,8 @@ true Debug;Release;MacOS;Linux AnyCPU - 4.2.1 - 4.2.1-pre1 + 4.2.2 + $(VersionPrefix)-pre1 PdbOnly diff --git a/UI.Avalonia/ViewModels/MainViewModel.cs b/UI.Avalonia/ViewModels/MainViewModel.cs index 64bd21f..f8be6c3 100644 --- a/UI.Avalonia/ViewModels/MainViewModel.cs +++ b/UI.Avalonia/ViewModels/MainViewModel.cs @@ -45,6 +45,18 @@ public MainViewModel(): this(new()){} [ObservableProperty] private bool? success; [ObservableProperty] private bool? validated; + private string[] AnalyzingMessages = + [ + "Analyzing the file structure", + "File structure analysis is taking longer than expected", + "Still analyzing the file structure", + "Yep, still analyzing the file structure", + "You wouldn't believe, but still analyzing", + "I can't believe it's taking so long", + "Hopefully it'll be over soon", + "How many files are there on this disc", + ]; + internal Dumper? dumper; private readonly SemaphoreSlim scanLock = new(1, 1); @@ -86,10 +98,10 @@ private void ResetViewModel() #pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed [RelayCommand] - private void ScanDiscs() => Dispatcher.UIThread.Post(() => ScanDiscsAsync(), DispatcherPriority.Background); + private void ScanDiscs() => ScanDiscsAsync(); [RelayCommand] - private void DumpDisc() => Dispatcher.UIThread.Post(() => DumpDiscAsync(), DispatcherPriority.Background); + private void DumpDisc() => DumpDiscAsync(); #pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed [RelayCommand] @@ -116,6 +128,8 @@ private async Task ScanDiscsAsync() StepSubtitle = "Checking the inserted disc…"; dumper?.Dispose(); dumper = new(); + await Task.Yield(); + try { dumper.DetectDisc("", @@ -152,6 +166,7 @@ private async Task ScanDiscsAsync() StepTitle = "Looking for a disc key"; StepSubtitle = "Checking IRD and Redump data sets…"; + await Task.Yield(); try { await dumper.FindDiscKeyAsync(settings.IrdDir).WaitAsync(dumper.Cts.Token).ConfigureAwait(false); @@ -225,7 +240,7 @@ private async Task DumpDiscAsync() StepTitle = "Dumping the disc"; StepSubtitle = "Decrypting and copying the data…"; - ProgressInfo = "Analyzing the file structure"; + ProgressInfo = AnalyzingMessages[0]; LastOperationSuccess = true; LastOperationWarning = false; LastOperationNotification = false; @@ -233,7 +248,8 @@ private async Task DumpDiscAsync() DumpingInProgress = true; CanEditSettings = false; EnableTaskbarProgress(); - + await Task.Yield(); + try { var threadCts = new CancellationTokenSource(); @@ -242,6 +258,10 @@ private async Task DumpDiscAsync() { try { + string[] dotsAnimation = ["", ".", "..", "..."]; + var animFrameIdx = 0; + var curAnalysisMsgIdx = 0; + var cnt = 0; do { if (dumper.TotalSectors > 0 && !dumper.Cts.IsCancellationRequested) @@ -251,7 +271,15 @@ private async Task DumpDiscAsync() Progress = (int)((dumper.ProcessedSectors + dumper.CurrentFileSector) * 10000L / dumper.TotalFileSectors); ProgressInfo = $"Sector data {(dumper.CurrentSector * dumper.SectorSize).AsStorageUnit()} of {(dumper.TotalSectors * dumper.SectorSize).AsStorageUnit()} / File {dumper.CurrentFileNumber} of {dumper.TotalFileCount}"; } - Task.Delay(200, combinedToken.Token).GetAwaiter().GetResult(); + else + { + ProgressInfo = $"{AnalyzingMessages[curAnalysisMsgIdx]}{dotsAnimation[animFrameIdx]}"; + if (++cnt % 5 is 0) + animFrameIdx = (animFrameIdx + 1) % dotsAnimation.Length; + if (cnt % 600 is 0) + curAnalysisMsgIdx = (curAnalysisMsgIdx + 1) % AnalyzingMessages.Length; + } + Task.Delay(100, combinedToken.Token).GetAwaiter().GetResult(); } while (!combinedToken.Token.IsCancellationRequested); } catch (TaskCanceledException) @@ -259,7 +287,7 @@ private async Task DumpDiscAsync() } }); monitor.Start(); - await dumper.DumpAsync(settings.OutputDir).WaitAsync(dumper.Cts.Token); + await dumper.DumpAsync(settings.OutputDir).WaitAsync(dumper.Cts.Token).ConfigureAwait(false); await threadCts.CancelAsync().ConfigureAwait(false); monitor.Join(100); } @@ -284,6 +312,7 @@ private async Task DumpDiscAsync() CanEditSettings = true; DumperIsReady = false; FoundDisc = false; + await Task.Yield(); if (dumper.Cts.IsCancellationRequested || LastOperationSuccess is false || LastOperationWarning is true)