From ef1a16c155e0bdf1bf148911c16c1c1b4d17c3b8 Mon Sep 17 00:00:00 2001 From: Mikal Stordal Date: Sun, 3 Nov 2024 21:51:41 +0100 Subject: [PATCH] refactor: better handling of case sensitivity in paths across platforms and file systems It's better, but still not perfect. We now check the source/destination if they're case sensitive by probing the file system, and the API endpoints will use a less accurate check by default since the more accurate check requires probing, which we don't want to do in parallel on thousands of files unless told to do it. Also consolidated the `IsLinux` and `IsRunningOnLinuxOrMac()` helpers into a single `IsLinuxOrMac` helper. --- .../API/v3/Controllers/FileController.cs | 26 +++- .../v3/Controllers/ImportFolderController.cs | 26 +++- .../API/v3/Controllers/RenamerController.cs | 3 +- .../API/v3/Controllers/SeriesController.cs | 16 ++- Shoko.Server/FileHelper/Hasher.cs | 4 +- Shoko.Server/FileHelper/MD4Managed.cs | 2 +- Shoko.Server/Renamer/RenameFileService.cs | 7 +- .../Cached/ImportFolderRepository.cs | 14 +- .../Scheduling/Jobs/Shoko/HashFileJob.cs | 2 +- Shoko.Server/Server/ShokoServer.cs | 2 +- Shoko.Server/Services/ActionService.cs | 30 ++-- .../Services/VideoLocal_PlaceService.cs | 13 +- .../Migration/ServerSettings_Legacy.cs | 2 +- Shoko.Server/Utilities/AVDumpHelper.cs | 4 +- .../RecoveringFileSystemWatcher.cs | 2 +- .../Utilities/MediaInfoLib/MediaInfo.cs | 2 +- Shoko.Server/Utilities/Utils.cs | 131 ++++++++++++++++-- 17 files changed, 229 insertions(+), 57 deletions(-) diff --git a/Shoko.Server/API/v3/Controllers/FileController.cs b/Shoko.Server/API/v3/Controllers/FileController.cs index 5016d5e0e..303bb54d5 100644 --- a/Shoko.Server/API/v3/Controllers/FileController.cs +++ b/Shoko.Server/API/v3/Controllers/FileController.cs @@ -1354,12 +1354,14 @@ private static void CheckXRefsForFile(SVR_VideoLocal file, ModelStateDictionary /// Set to true to include series and episode cross-references. /// Include data from selected s. /// Limit the number of returned results. + /// Set to true to skip the per directory comparison logic and + /// instead rely on the global platform comparison based on the currently running platform. /// A list of all files with a file location that ends with the given path. [HttpGet("PathEndsWith")] public ActionResult> PathEndsWithQuery([FromQuery] string path, [FromQuery] bool includeXRefs = true, [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null, - [Range(0, 100)] int limit = 0) - => PathEndsWithInternal(path, includeXRefs, includeDataFrom, limit); + [Range(0, 100)] int limit = 0, [FromQuery] bool quickCompare = false) + => PathEndsWithInternal(path, includeXRefs, includeDataFrom, limit, quickCompare); /// /// Search for a file by path or name. Internally, it will convert forward @@ -1370,12 +1372,14 @@ public ActionResult> PathEndsWithQuery([FromQuery] string path, [From /// Set to true to include series and episode cross-references. /// Include data from selected s. /// Limit the number of returned results. + /// Set to true to skip the per directory comparison logic and + /// instead rely on the global platform comparison based on the currently running platform. /// A list of all files with a file location that ends with the given path. [HttpGet("PathEndsWith/{*path}")] public ActionResult> PathEndsWithPath([FromRoute] string path, [FromQuery] bool includeXRefs = true, [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet includeDataFrom = null, - [Range(0, 100)] int limit = 0) - => PathEndsWithInternal(Uri.UnescapeDataString(path), includeXRefs, includeDataFrom, limit); + [Range(0, 100)] int limit = 0, [FromQuery] bool quickCompare = true) + => PathEndsWithInternal(Uri.UnescapeDataString(path), includeXRefs, includeDataFrom, limit, quickCompare); /// /// Search for a file by path or name. Internally, it will convert forward @@ -1386,10 +1390,12 @@ public ActionResult> PathEndsWithPath([FromRoute] string path, [FromQ /// Set to true to include series and episode cross-references. /// Include data from selected s. /// Limit the number of returned results. + /// Set to true to skip the per directory comparison logic and + /// instead rely on the global platform comparison based on the currently running platform. /// A list of all files with a file location that ends with the given path. [NonAction] private ActionResult> PathEndsWithInternal(string path, bool includeXRefs, - HashSet includeDataFrom, int limit = 0) + HashSet includeDataFrom, int limit = 0, bool quickCompare = true) { if (string.IsNullOrWhiteSpace(path)) return new List(); @@ -1399,7 +1405,15 @@ private ActionResult> PathEndsWithInternal(string path, bool includeX .Replace('\\', Path.DirectorySeparatorChar); var results = RepoFactory.VideoLocalPlace.GetAll() .AsParallel() - .Where(location => location.FullServerPath?.EndsWith(query, StringComparison.OrdinalIgnoreCase) ?? false) + .Where(location => + { + var serverPath = location.FullServerPath; + if (string.IsNullOrEmpty(serverPath)) + return false; + + var comparison = quickCompare ? Utils.PlatformComparison : Utils.GetComparisonFor(Path.GetDirectoryName(serverPath)); + return serverPath.EndsWith(query, comparison); + }) .Select(location => location.VideoLocal) .Where(file => { diff --git a/Shoko.Server/API/v3/Controllers/ImportFolderController.cs b/Shoko.Server/API/v3/Controllers/ImportFolderController.cs index 92b0ac21b..613407b5f 100644 --- a/Shoko.Server/API/v3/Controllers/ImportFolderController.cs +++ b/Shoko.Server/API/v3/Controllers/ImportFolderController.cs @@ -12,6 +12,7 @@ using Shoko.Server.Repositories; using Shoko.Server.Services; using Shoko.Server.Settings; +using Shoko.Server.Utilities; namespace Shoko.Server.API.v3.Controllers; @@ -53,7 +54,17 @@ public ActionResult AddImportFolder([FromBody] ImportFolder folder if (!Directory.Exists(folder.Path)) return ValidationProblem("Path does not exist. Import Folders must be a location that exists on the server.", nameof(folder.Path)); - if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF => folder.Path.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.Path, StringComparison.OrdinalIgnoreCase))) + if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF => + { + var comparison = Utils.GetComparisonFor(folder.Path, iF.ImportFolderLocation); + var newLocation = folder.Path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (newLocation[^1] != Path.DirectorySeparatorChar) + newLocation += Path.DirectorySeparatorChar; + var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (existingLocation[^1] != Path.DirectorySeparatorChar) + existingLocation += Path.DirectorySeparatorChar; + return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison); + })) return ValidationProblem("Unable to nest an import folder within another import folder."); try @@ -137,8 +148,17 @@ public ActionResult EditImportFolder([FromBody] ImportFolder folder) if (!string.IsNullOrEmpty(folder.Path) && !Directory.Exists(folder.Path)) ModelState.AddModelError(nameof(folder.Path), "Path does not exist. Import Folders must be a location that exists on the server."); - if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID) - .Any(iF => folder.Path.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.Path, StringComparison.OrdinalIgnoreCase))) + if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF => + { + var comparison = Utils.GetComparisonFor(folder.Path, iF.ImportFolderLocation); + var newLocation = folder.Path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (newLocation[^1] != Path.DirectorySeparatorChar) + newLocation += Path.DirectorySeparatorChar; + var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (existingLocation[^1] != Path.DirectorySeparatorChar) + existingLocation += Path.DirectorySeparatorChar; + return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison); + })) ModelState.AddModelError(nameof(folder.Path), "Unable to nest an import folder within another import folder."); if (folder.ID == 0) diff --git a/Shoko.Server/API/v3/Controllers/RenamerController.cs b/Shoko.Server/API/v3/Controllers/RenamerController.cs index 1df473ab9..5f4b4a6ef 100644 --- a/Shoko.Server/API/v3/Controllers/RenamerController.cs +++ b/Shoko.Server/API/v3/Controllers/RenamerController.cs @@ -588,7 +588,8 @@ public async Task> DirectlyRelocateFileLocation([ // Sanitize relative path and reject paths leading to outside the import folder. var fullPath = Path.GetFullPath(Path.Combine(importFolder.ImportFolderLocation, body.RelativePath)); - if (!fullPath.StartsWith(importFolder.ImportFolderLocation, StringComparison.OrdinalIgnoreCase)) + // We use ordinal here since we want to compare the exact path with what we got from the import folder to what the final path ends up being. + if (!fullPath.StartsWith(importFolder.ImportFolderLocation, StringComparison.Ordinal)) return BadRequest("The provided relative path leads outside the import folder."); var sanitizedRelativePath = Path.GetRelativePath(importFolder.ImportFolderLocation, fullPath); diff --git a/Shoko.Server/API/v3/Controllers/SeriesController.cs b/Shoko.Server/API/v3/Controllers/SeriesController.cs index c1770e230..c1a090fc7 100644 --- a/Shoko.Server/API/v3/Controllers/SeriesController.cs +++ b/Shoko.Server/API/v3/Controllers/SeriesController.cs @@ -2821,11 +2821,14 @@ public ActionResult MoveSeries([FromRoute, Range(1, int.MaxValue)] int seriesID, } /// - /// Get the series that reside in the path that ends with + /// Get the series that reside in the path that ends with . /// + /// The path to search for. + /// Set to true to skip the per directory comparison logic and + /// instead rely on the global platform comparison based on the currently running platform. /// [HttpGet("PathEndsWith/{*path}")] - public ActionResult> PathEndsWith([FromRoute] string path) + public ActionResult> PathEndsWith([FromRoute] string path, bool quickCompare = true) { var user = User; var query = path; @@ -2845,9 +2848,14 @@ public ActionResult> PathEndsWith([FromRoute] string path) return RepoFactory.VideoLocalPlace.GetAll() .Where(a => { - if (a.FullServerPath == null) return false; + if (a.FullServerPath == null) + return false; var dir = Path.GetDirectoryName(a.FullServerPath); - return dir != null && dir.EndsWith(query, StringComparison.OrdinalIgnoreCase); + if (string.IsNullOrEmpty(dir)) + return false; + + var comparison = quickCompare ? Utils.PlatformComparison : Utils.GetComparisonFor(dir); + return dir != null && dir.EndsWith(query, comparison); }) .SelectMany(a => a.VideoLocal?.AnimeEpisodes ?? Enumerable.Empty()) .DistinctBy(a => a.AnimeSeriesID) diff --git a/Shoko.Server/FileHelper/Hasher.cs b/Shoko.Server/FileHelper/Hasher.cs index 1073b5ee4..18286d6ad 100644 --- a/Shoko.Server/FileHelper/Hasher.cs +++ b/Shoko.Server/FileHelper/Hasher.cs @@ -140,7 +140,7 @@ public static string GetVersion() public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgress, bool getCRC32, bool getMD5, bool getSHA1) { var rhash = new Hashes(); - if (Finalise.ModuleHandle == IntPtr.Zero && !Utils.IsLinux) + if (Finalise.ModuleHandle == IntPtr.Zero && !Utils.IsLinuxOrMac) return CalculateHashes_here(strPath, onHashProgress, getCRC32, getMD5, getSHA1); var hash = new byte[56]; @@ -149,7 +149,7 @@ public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgre try { var filename = strPath; - if (!Utils.IsLinux) + if (!Utils.IsLinuxOrMac) { filename = strPath.StartsWith(@"\\") ? strPath diff --git a/Shoko.Server/FileHelper/MD4Managed.cs b/Shoko.Server/FileHelper/MD4Managed.cs index 52b152973..fd2e4efb3 100644 --- a/Shoko.Server/FileHelper/MD4Managed.cs +++ b/Shoko.Server/FileHelper/MD4Managed.cs @@ -22,7 +22,7 @@ protected MD4() { var obj = CryptoConfig.CreateFromName(hashName); // in case machine.config isn't configured to use any MD4 implementation - if (obj == null || Utils.IsRunningOnLinuxOrMac()) + if (obj == null || Utils.IsLinuxOrMac) { obj = new MD4Managed(); } diff --git a/Shoko.Server/Renamer/RenameFileService.cs b/Shoko.Server/Renamer/RenameFileService.cs index 522ce4486..3127132a7 100644 --- a/Shoko.Server/Renamer/RenameFileService.cs +++ b/Shoko.Server/Renamer/RenameFileService.cs @@ -240,14 +240,15 @@ private static RelocationResult UnAbstractResult(SVR_VideoLocal_Place place, Abs var newRelativeDirectory = shouldMove && !result.SkipMove ? result.Path : Path.GetDirectoryName(place.FilePath); var newRelativePath = !string.IsNullOrEmpty(newRelativeDirectory) && newRelativeDirectory.Length > 0 ? Path.Combine(newRelativeDirectory, newFileName) : newFileName; var newFullPath = Path.Combine(newImportFolder.Path, newRelativePath); + var oldFullPath = place.FullServerPath; + var comparison = Utils.GetComparisonFor(Path.GetDirectoryName(newFullPath), Path.GetDirectoryName(oldFullPath)); return new() { Success = true, ImportFolder = newImportFolder, RelativePath = newRelativePath, - // TODO: Handle file-systems that are or aren't case sensitive. - Renamed = !string.Equals(place.FileName, result.FileName, StringComparison.OrdinalIgnoreCase), - Moved = !string.Equals(Path.GetDirectoryName(place.FullServerPath), Path.GetDirectoryName(newFullPath), StringComparison.OrdinalIgnoreCase), + Renamed = !string.Equals(place.FileName, result.FileName, comparison), + Moved = !string.Equals(Path.GetDirectoryName(oldFullPath), Path.GetDirectoryName(newFullPath), comparison), }; } diff --git a/Shoko.Server/Repositories/Cached/ImportFolderRepository.cs b/Shoko.Server/Repositories/Cached/ImportFolderRepository.cs index e97848135..bfcb60be0 100644 --- a/Shoko.Server/Repositories/Cached/ImportFolderRepository.cs +++ b/Shoko.Server/Repositories/Cached/ImportFolderRepository.cs @@ -4,6 +4,7 @@ using Shoko.Models.Server; using Shoko.Server.Databases; using Shoko.Server.Models; +using Shoko.Server.Utilities; namespace Shoko.Server.Repositories.Cached; @@ -73,10 +74,19 @@ public SVR_ImportFolder SaveImportFolder(ImportFolder folder) throw new Exception("Cannot find Import Folder location"); } - if (GetAll().ExceptBy([folder.ImportFolderID], iF => iF.ImportFolderID).Any(iF => folder.ImportFolderLocation.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.ImportFolderLocation, StringComparison.OrdinalIgnoreCase))) + if (GetAll().ExceptBy([folder.ImportFolderID], iF => iF.ImportFolderID).Any(iF => + { + var comparison = Utils.GetComparisonFor(folder.ImportFolderLocation, iF.ImportFolderLocation); + var newLocation = folder.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (newLocation[^1] != Path.DirectorySeparatorChar) + newLocation += Path.DirectorySeparatorChar; + var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + if (existingLocation[^1] != Path.DirectorySeparatorChar) + existingLocation += Path.DirectorySeparatorChar; + return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison); + })) throw new Exception("Unable to nest an import folder within another import folder."); - ns.ImportFolderName = folder.ImportFolderName; ns.ImportFolderLocation = folder.ImportFolderLocation; ns.IsDropDestination = folder.IsDropDestination; diff --git a/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs b/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs index 708b9b207..c219ae57b 100644 --- a/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs +++ b/Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs @@ -277,7 +277,7 @@ private bool HandleReadOnlyException(Exception ex) var info = new FileInfo(FilePath); if (info.IsReadOnly) info.IsReadOnly = false; - if (!info.IsReadOnly && !Utils.IsRunningOnLinuxOrMac()) + if (!info.IsReadOnly && !Utils.IsLinuxOrMac) { return true; } diff --git a/Shoko.Server/Server/ShokoServer.cs b/Shoko.Server/Server/ShokoServer.cs index a68aca68d..06c976de2 100644 --- a/Shoko.Server/Server/ShokoServer.cs +++ b/Shoko.Server/Server/ShokoServer.cs @@ -117,7 +117,7 @@ public bool StartUpServer() private bool CheckBlockedFiles() { - if (Utils.IsRunningOnLinuxOrMac()) return true; + if (Utils.IsLinuxOrMac) return true; var programlocation = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location); diff --git a/Shoko.Server/Services/ActionService.cs b/Shoko.Server/Services/ActionService.cs index c71bfb1bf..409c779a3 100644 --- a/Shoko.Server/Services/ActionService.cs +++ b/Shoko.Server/Services/ActionService.cs @@ -162,12 +162,14 @@ public async Task RunImport_ScanFolder(int importFolderID, bool skipMyList = fal dictFilesExisting[vl.FullServerPath] = vl; } - Utils.GetFilesForImportFolder(folder.BaseDirectory, ref fileList); + var baseDirectory = folder.BaseDirectory; + var comparer = Utils.GetComparerFor(baseDirectory.FullName); + Utils.GetFilesForImportFolder(baseDirectory, ref fileList); // Get Ignored Files and remove them from the scan listing var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos().SelectMany(a => a.Places) .Select(a => a.FullServerPath).Where(a => !string.IsNullOrEmpty(a)).ToList(); - fileList = fileList.Except(ignoredFiles, StringComparer.InvariantCultureIgnoreCase).ToList(); + fileList = fileList.Except(ignoredFiles, comparer).ToList(); // get a list of all files in the share foreach (var fileName in fileList) @@ -210,19 +212,27 @@ public async Task RunImport_DropFolders() { var settings = _settingsProvider.GetSettings(); var scheduler = await _schedulerFactory.GetScheduler(); + // Get Ignored Files and remove them from the scan listing + var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos() + .SelectMany(a => a.Places) + .Select(a => a.FullServerPath) + .Where(a => !string.IsNullOrEmpty(a)) + .ToList(); + // get a complete list of files var fileList = new List(); - foreach (var share in RepoFactory.ImportFolder.GetAll()) + foreach (var folder in RepoFactory.ImportFolder.GetAll()) { - if (!share.FolderIsDropSource) continue; - Utils.GetFilesForImportFolder(share.BaseDirectory, ref fileList); + if (!folder.FolderIsDropSource) continue; + + var fileListForFolder = new List(); + var baseDirectory = folder.BaseDirectory; + var comparer = Utils.GetComparerFor(baseDirectory.FullName); + Utils.GetFilesForImportFolder(baseDirectory, ref fileListForFolder); + fileListForFolder = fileListForFolder.Except(ignoredFiles, comparer).ToList(); + fileList.AddRange(fileListForFolder); } - // Get Ignored Files and remove them from the scan listing - var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos().SelectMany(a => a.Places) - .Select(a => a.FullServerPath).Where(a => !string.IsNullOrEmpty(a)).ToList(); - fileList = fileList.Except(ignoredFiles, StringComparer.InvariantCultureIgnoreCase).ToList(); - // get a list of all the shares we are looking at int filesFound = 0, videosFound = 0; var i = 0; diff --git a/Shoko.Server/Services/VideoLocal_PlaceService.cs b/Shoko.Server/Services/VideoLocal_PlaceService.cs index bd174b59e..51c80ec5e 100644 --- a/Shoko.Server/Services/VideoLocal_PlaceService.cs +++ b/Shoko.Server/Services/VideoLocal_PlaceService.cs @@ -106,7 +106,8 @@ public async Task DirectlyRelocateFile(SVR_VideoLocal_Place pl // Sanitize relative path and reject paths leading to outside the import folder. var fullPath = Path.GetFullPath(Path.Combine(request.ImportFolder.Path, request.RelativePath)); - if (!fullPath.StartsWith(request.ImportFolder.Path, StringComparison.OrdinalIgnoreCase)) + // We use ordinal here since we want to compare the exact path with what we got from the import folder to what the final path ends up being. + if (!fullPath.StartsWith(request.ImportFolder.Path, StringComparison.Ordinal)) return new() { Success = false, @@ -193,11 +194,12 @@ public async Task DirectlyRelocateFile(SVR_VideoLocal_Place pl var newFolderPath = Path.GetDirectoryName(newRelativePath); var newFullPath = Path.Combine(request.ImportFolder.Path, newRelativePath); var newFileName = Path.GetFileName(newRelativePath); - var renamed = !string.Equals(Path.GetFileName(oldRelativePath), newFileName, StringComparison.OrdinalIgnoreCase); - var moved = !string.Equals(Path.GetDirectoryName(oldFullPath), Path.GetDirectoryName(newFullPath), StringComparison.OrdinalIgnoreCase); + var comparison = Utils.GetComparisonFor(Path.GetDirectoryName(newFullPath), Path.GetDirectoryName(oldFullPath)); + var renamed = !string.Equals(Path.GetFileName(oldRelativePath), newFileName, comparison); + var moved = !string.Equals(Path.GetDirectoryName(oldFullPath), Path.GetDirectoryName(newFullPath), comparison); // Last ditch effort to ensure we aren't moving a file unto itself - if (string.Equals(newFullPath, oldFullPath, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(newFullPath, oldFullPath, comparison)) { _logger.LogTrace("Resolved to relocate {FilePath} onto itself. Nothing to do.", newFullPath); return new() @@ -612,6 +614,7 @@ private void MoveExternalSubtitles(string newFullServerPath, string oldFullServe string.IsNullOrEmpty(oldFileName) || string.IsNullOrEmpty(newFileName)) return; + var comparison = Utils.GetComparisonFor(Path.GetDirectoryName(newFullServerPath), Path.GetDirectoryName(oldFullServerPath)); var textStreams = SubtitleHelper.GetSubtitleStreams(oldFullServerPath); // move any subtitle files foreach (var subtitleFile in textStreams) @@ -628,7 +631,7 @@ private void MoveExternalSubtitles(string newFullServerPath, string oldFullServe } var newSubPath = Path.Combine(newParent, newFileName + subtitleFile.Filename[oldFileName.Length..]); - if (string.Equals(subPath, newSubPath, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(subPath, newSubPath, comparison)) { _logger.LogDebug("Attempting to move subtitle file onto itself. Skipping. Path: {FilePath} to {FilePath}", subPath, newSubPath); continue; diff --git a/Shoko.Server/Settings/Migration/ServerSettings_Legacy.cs b/Shoko.Server/Settings/Migration/ServerSettings_Legacy.cs index 1181734bf..c8382f240 100644 --- a/Shoko.Server/Settings/Migration/ServerSettings_Legacy.cs +++ b/Shoko.Server/Settings/Migration/ServerSettings_Legacy.cs @@ -25,7 +25,7 @@ private static string ApplicationPath { get { - if (Utils.IsLinux) + if (Utils.IsLinuxOrMac) { return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".shoko", diff --git a/Shoko.Server/Utilities/AVDumpHelper.cs b/Shoko.Server/Utilities/AVDumpHelper.cs index b35fcde18..a683e1fa3 100644 --- a/Shoko.Server/Utilities/AVDumpHelper.cs +++ b/Shoko.Server/Utilities/AVDumpHelper.cs @@ -31,7 +31,7 @@ public static partial class AVDumpHelper private const string AVDumpURL = @"AVD3_URL_GOES_HERE"; - private static readonly string AVDumpExecutable = Path.Combine(WorkingDirectory, Utils.IsRunningOnLinuxOrMac() ? "AVDump3CL.dll" : "AVDump3CL.exe"); + private static readonly string AVDumpExecutable = Path.Combine(WorkingDirectory, Utils.IsLinuxOrMac ? "AVDump3CL.dll" : "AVDump3CL.exe"); private static readonly ConcurrentDictionary ActiveSessions = new(); @@ -479,7 +479,7 @@ private static Process GetSubProcessForOS(params string[] argumentList) CreateNoWindow = true }; - if (Utils.IsRunningOnLinuxOrMac()) + if (Utils.IsLinuxOrMac) { startInfo.FileName = "dotnet"; startInfo.ArgumentList.Add(AVDumpExecutable); diff --git a/Shoko.Server/Utilities/FileSystemWatcher/RecoveringFileSystemWatcher.cs b/Shoko.Server/Utilities/FileSystemWatcher/RecoveringFileSystemWatcher.cs index 8abc1ce1a..b26a17db1 100644 --- a/Shoko.Server/Utilities/FileSystemWatcher/RecoveringFileSystemWatcher.cs +++ b/Shoko.Server/Utilities/FileSystemWatcher/RecoveringFileSystemWatcher.cs @@ -386,7 +386,7 @@ private long CanAccessFile(string fileName, ref Exception e) if (info.IsReadOnly) info.IsReadOnly = false; // check to see if it stuck. On linux, we can't just WinAPI hack our way out, so don't recurse in that case, anyway - if (!new FileInfo(fileName).IsReadOnly && !Utils.IsRunningOnLinuxOrMac()) return GetFileSize(fileName, accessType); + if (!new FileInfo(fileName).IsReadOnly && !Utils.IsLinuxOrMac) return GetFileSize(fileName, accessType); } catch { diff --git a/Shoko.Server/Utilities/MediaInfoLib/MediaInfo.cs b/Shoko.Server/Utilities/MediaInfoLib/MediaInfo.cs index 6251ff280..48fb8ec58 100644 --- a/Shoko.Server/Utilities/MediaInfoLib/MediaInfo.cs +++ b/Shoko.Server/Utilities/MediaInfoLib/MediaInfo.cs @@ -132,7 +132,7 @@ private static string GetMediaInfoPathForOS() path = settings.Import.MediaInfoPath; if (!string.IsNullOrEmpty(path) && File.Exists(path)) return path; - if (Utils.IsRunningOnLinuxOrMac()) return "mediainfo"; + if (Utils.IsLinuxOrMac) return "mediainfo"; var exePath = Assembly.GetEntryAssembly()?.Location; var exeDir = Path.GetDirectoryName(exePath); diff --git a/Shoko.Server/Utilities/Utils.cs b/Shoko.Server/Utilities/Utils.cs index 6550011b5..0601ef5a8 100644 --- a/Shoko.Server/Utilities/Utils.cs +++ b/Shoko.Server/Utilities/Utils.cs @@ -19,8 +19,8 @@ using Quartz.Logging; using Shoko.Models.Enums; using Shoko.Server.API.SignalR.NLog; -using Shoko.Server.Providers.AniDB.Titles; using Shoko.Server.Server; +using Shoko.Server.Services; using Shoko.Server.Settings; namespace Shoko.Server.Utilities; @@ -46,7 +46,7 @@ public static string ApplicationPath if (!string.IsNullOrWhiteSpace(shokoHome)) return _applicationPath = Path.GetFullPath(shokoHome); - if (IsLinux) + if (IsLinuxOrMac) return _applicationPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".shoko", DefaultInstance); @@ -133,7 +133,7 @@ public static void InitLogger() } LogProvider.SetLogProvider(new NLog.Extensions.Logging.NLogLoggerFactory()); - + LogManager.ReconfigExistingLoggers(); } @@ -414,20 +414,125 @@ public static bool IsDirectoryWritable(string dirPath, bool throwIfFails = false } } - public static bool IsLinux + public static bool IsLinuxOrMac + => (int)Environment.OSVersion.Platform is 4 or 6 or 128; + + public static StringComparison PlatformComparison { get; private set; } = IsLinuxOrMac + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + /// + /// Returns the correct for the given file system directories. This is useful for string operations that need to be case-sensitive or insensitive depending on the underlying file systems. + /// + /// The paths to the directories to check. + /// The correct for the given file system directories. + public static StringComparison GetComparisonFor(params string[] directoryPaths) => + directoryPaths.Any(directoryPath => IsFileSystemCaseSensitive(directoryPath) ?? IsLinuxOrMac) ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + + /// + /// Returns the correct for the given file system directories. This is useful for string operations that need to be case-sensitive or insensitive depending on the underlying file systems. + /// + /// The paths to the directories to check. + /// The correct for the given file system directories. + public static StringComparer GetComparerFor(params string[] directoryPaths) => + directoryPaths.Any(directoryPath => IsFileSystemCaseSensitive(directoryPath) ?? IsLinuxOrMac) ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; + + // https://referencesource.microsoft.com/mscorlib/microsoft/win32/win32native.cs.html#9f6ca3226ff8f9ba + // https://referencesource.microsoft.com/mscorlib/microsoft/win32/win32native.cs.html#dd35d7f626262141 + private const int ERROR_FILE_EXISTS = unchecked((int)0x80070050); + + /// + /// Check whether the operating system handles file names case-sensitive in + /// the specified directory or the the first existing parent directory of + /// the specified directory. + /// + /// + /// Modified from; + /// https://stackoverflow.com/questions/430256/how-do-i-determine-whether-the-filesystem-is-case-sensitive-in-net/53228078#53228078 + /// + /// The path to the directory to check. + /// A value indicating whether the operating system handles file names case-sensitive in the specified directory. + /// is null. + /// contains one or more invalid characters. + /// The specified directory does not exist. + /// The current user has no write permission to the specified directory. + private static bool? IsFileSystemCaseSensitive(string directoryPath) { - get + try { - var p = (int)Environment.OSVersion.Platform; - return p == 4 || p == 6 || p == 128; - } - } + while (!string.IsNullOrEmpty(directoryPath) && !Directory.Exists(directoryPath)) + directoryPath = Path.GetDirectoryName(directoryPath); - public static bool IsRunningOnLinuxOrMac() - { - return !RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - } + if (string.IsNullOrEmpty(directoryPath)) + return null; + var fileWatcherService = Utils.ServiceContainer.GetRequiredService(); + while (true) + { + var fileNameLower = ".cstest." + Guid.NewGuid().ToString(); + var fileNameUpper = fileNameLower.ToUpperInvariant(); + var filePathLower = Path.Combine(directoryPath, fileNameLower); + var filePathUpper = Path.Combine(directoryPath, fileNameUpper); + FileStream fileStreamLower = null; + FileStream fileStreamUpper = null; + try + { + fileWatcherService.AddFileWatcherExclusion(filePathLower); + fileWatcherService.AddFileWatcherExclusion(filePathUpper); + try + { + // Try to create filePathUpper to ensure a unique non-existing file. + fileStreamUpper = new FileStream(filePathUpper, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.DeleteOnClose); + + // After ensuring that it didn't exist before, filePathUpper must be closed/deleted again to ensure correct opening of filePathLower, regardless of the case-sensitivity of the file system. + // On case-sensitive file systems there is a tiny chance for a race condition, where another process could create filePathUpper between closing/deleting it here and newly creating it after filePathLower. + // This method would then incorrectly indicate a case-insensitive file system. + fileStreamUpper.Dispose(); + } + catch (IOException ioException) when (ioException.HResult is ERROR_FILE_EXISTS) + { + // filePathUpper already exists, try another file name + continue; + } + + try + { + fileStreamLower = new FileStream(filePathLower, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.DeleteOnClose); + } + catch (IOException ioException) when (ioException.HResult is ERROR_FILE_EXISTS) + { + // filePathLower already exists, try another file name + continue; + } + + try + { + fileStreamUpper = new FileStream(filePathUpper, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.DeleteOnClose); + + // filePathUpper does not exist, this indicates case-sensitivity + return true; + } + catch (IOException ioException) when (ioException.HResult is ERROR_FILE_EXISTS) + { + // fileNameUpper already exists, this indicates case-insensitivity + return false; + } + } + finally + { + fileStreamLower?.Dispose(); + fileStreamUpper?.Dispose(); + fileWatcherService.RemoveFileWatcherExclusion(filePathLower); + fileWatcherService.RemoveFileWatcherExclusion(filePathUpper); + } + } + } + catch (Exception ex) + { + _logger.Error(ex, $"Failed to determine file system case-sensitivity for directory {directoryPath}"); + return null; + } + } /// /// Determines an encoded string's encoding by analyzing its byte order mark (BOM). /// Defaults to ASCII when detection of the text file's endianness fails.