refactor: better handling of case sensitivity in paths across platfor…

…ms and file systems

It's better, but still not perfect.

We now check the source/destination if they're case sensitive by probing the file system, and the API endpoints will use a less accurate check by default since the more accurate check requires probing, which we don't want to do in parallel on thousands of files unless told to do it.

Also consolidated the `IsLinux` and `IsRunningOnLinuxOrMac()` helpers into a single `IsLinuxOrMac` helper.

Shoko.Server/API/v3/Controllers/FileController.cs

@@ -1354,12 +1354,14 @@ private static void CheckXRefsForFile(SVR_VideoLocal file, ModelStateDictionary
     /// <param name="includeXRefs">Set to true to include series and episode cross-references.</param>
     /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
     /// <param name="limit">Limit the number of returned results.</param>
+    /// <param name="quickCompare">Set to true to skip the per directory comparison logic and
+    /// instead rely on the global platform comparison based on the currently running platform.</param>
     /// <returns>A list of all files with a file location that ends with the given path.</returns>
     [HttpGet("PathEndsWith")]
     public ActionResult<List<File>> PathEndsWithQuery([FromQuery] string path, [FromQuery] bool includeXRefs = true,
         [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet<DataSource> includeDataFrom = null,
-        [Range(0, 100)] int limit = 0)
-        => PathEndsWithInternal(path, includeXRefs, includeDataFrom, limit);
+        [Range(0, 100)] int limit = 0, [FromQuery] bool quickCompare = false)
+        => PathEndsWithInternal(path, includeXRefs, includeDataFrom, limit, quickCompare);
 
     /// <summary>
     /// Search for a file by path or name. Internally, it will convert forward
@@ -1370,12 +1372,14 @@ public ActionResult<List<File>> PathEndsWithQuery([FromQuery] string path, [From
     /// <param name="includeXRefs">Set to true to include series and episode cross-references.</param>
     /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
     /// <param name="limit">Limit the number of returned results.</param>
+    /// <param name="quickCompare">Set to true to skip the per directory comparison logic and
+    /// instead rely on the global platform comparison based on the currently running platform.</param>
     /// <returns>A list of all files with a file location that ends with the given path.</returns>
     [HttpGet("PathEndsWith/{*path}")]
     public ActionResult<List<File>> PathEndsWithPath([FromRoute] string path, [FromQuery] bool includeXRefs = true,
         [FromQuery, ModelBinder(typeof(CommaDelimitedModelBinder))] HashSet<DataSource> includeDataFrom = null,
-        [Range(0, 100)] int limit = 0)
-        => PathEndsWithInternal(Uri.UnescapeDataString(path), includeXRefs, includeDataFrom, limit);
+        [Range(0, 100)] int limit = 0, [FromQuery] bool quickCompare = true)
+        => PathEndsWithInternal(Uri.UnescapeDataString(path), includeXRefs, includeDataFrom, limit, quickCompare);
 
     /// <summary>
     /// Search for a file by path or name. Internally, it will convert forward
@@ -1386,10 +1390,12 @@ public ActionResult<List<File>> PathEndsWithPath([FromRoute] string path, [FromQ
     /// <param name="includeXRefs">Set to true to include series and episode cross-references.</param>
     /// <param name="includeDataFrom">Include data from selected <see cref="DataSource"/>s.</param>
     /// <param name="limit">Limit the number of returned results.</param>
+    /// <param name="quickCompare">Set to true to skip the per directory comparison logic and
+    /// instead rely on the global platform comparison based on the currently running platform.</param>
     /// <returns>A list of all files with a file location that ends with the given path.</returns>
     [NonAction]
     private ActionResult<List<File>> PathEndsWithInternal(string path, bool includeXRefs,
-        HashSet<DataSource> includeDataFrom, int limit = 0)
+        HashSet<DataSource> includeDataFrom, int limit = 0, bool quickCompare = true)
     {
         if (string.IsNullOrWhiteSpace(path))
             return new List<File>();
@@ -1399,7 +1405,15 @@ private ActionResult<List<File>> PathEndsWithInternal(string path, bool includeX
             .Replace('\\', Path.DirectorySeparatorChar);
         var results = RepoFactory.VideoLocalPlace.GetAll()
             .AsParallel()
-            .Where(location => location.FullServerPath?.EndsWith(query, StringComparison.OrdinalIgnoreCase) ?? false)
+            .Where(location =>
+            {
+                var serverPath = location.FullServerPath;
+                if (string.IsNullOrEmpty(serverPath))
+                    return false;
+
+                var comparison = quickCompare ? Utils.PlatformComparison : Utils.GetComparisonFor(Path.GetDirectoryName(serverPath));
+                return serverPath.EndsWith(query, comparison);
+            })
             .Select(location => location.VideoLocal)
             .Where(file =>
             {

Shoko.Server/API/v3/Controllers/ImportFolderController.cs

@@ -12,6 +12,7 @@
 using Shoko.Server.Repositories;
 using Shoko.Server.Services;
 using Shoko.Server.Settings;
+using Shoko.Server.Utilities;
 
 namespace Shoko.Server.API.v3.Controllers;
 
@@ -53,7 +54,17 @@ public ActionResult<ImportFolder> AddImportFolder([FromBody] ImportFolder folder
         if (!Directory.Exists(folder.Path))
             return ValidationProblem("Path does not exist. Import Folders must be a location that exists on the server.", nameof(folder.Path));
 
-        if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF => folder.Path.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.Path, StringComparison.OrdinalIgnoreCase)))
+        if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF =>
+        {
+            var comparison = Utils.GetComparisonFor(folder.Path, iF.ImportFolderLocation);
+            var newLocation = folder.Path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (newLocation[^1] != Path.DirectorySeparatorChar)
+                newLocation += Path.DirectorySeparatorChar;
+            var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (existingLocation[^1] != Path.DirectorySeparatorChar)
+                existingLocation += Path.DirectorySeparatorChar;
+            return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison);
+        }))
             return ValidationProblem("Unable to nest an import folder within another import folder.");
 
         try
@@ -137,8 +148,17 @@ public ActionResult EditImportFolder([FromBody] ImportFolder folder)
         if (!string.IsNullOrEmpty(folder.Path) && !Directory.Exists(folder.Path))
             ModelState.AddModelError(nameof(folder.Path), "Path does not exist. Import Folders must be a location that exists on the server.");
 
-        if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID)
-            .Any(iF => folder.Path.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.Path, StringComparison.OrdinalIgnoreCase)))
+        if (RepoFactory.ImportFolder.GetAll().ExceptBy([folder.ID], iF => iF.ImportFolderID).Any(iF =>
+        {
+            var comparison = Utils.GetComparisonFor(folder.Path, iF.ImportFolderLocation);
+            var newLocation = folder.Path.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (newLocation[^1] != Path.DirectorySeparatorChar)
+                newLocation += Path.DirectorySeparatorChar;
+            var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (existingLocation[^1] != Path.DirectorySeparatorChar)
+                existingLocation += Path.DirectorySeparatorChar;
+            return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison);
+        }))
             ModelState.AddModelError(nameof(folder.Path), "Unable to nest an import folder within another import folder.");
 
         if (folder.ID == 0)

Shoko.Server/API/v3/Controllers/RenamerController.cs

@@ -588,7 +588,8 @@ public async Task<ActionResult<RelocationResult>> DirectlyRelocateFileLocation([
 
         // Sanitize relative path and reject paths leading to outside the import folder.
         var fullPath = Path.GetFullPath(Path.Combine(importFolder.ImportFolderLocation, body.RelativePath));
-        if (!fullPath.StartsWith(importFolder.ImportFolderLocation, StringComparison.OrdinalIgnoreCase))
+        // We use ordinal here since we want to compare the exact path with what we got from the import folder to what the final path ends up being.
+        if (!fullPath.StartsWith(importFolder.ImportFolderLocation, StringComparison.Ordinal))
             return BadRequest("The provided relative path leads outside the import folder.");
         var sanitizedRelativePath = Path.GetRelativePath(importFolder.ImportFolderLocation, fullPath);
 

Shoko.Server/API/v3/Controllers/SeriesController.cs

@@ -2821,11 +2821,14 @@ public ActionResult MoveSeries([FromRoute, Range(1, int.MaxValue)] int seriesID,
     }
 
     /// <summary>
-    /// Get the series that reside in the path that ends with <param name="path"></param>
+    /// Get the series that reside in the path that ends with <paramref name="path"/>.
     /// </summary>
+    /// <param name="path">The path to search for.</param>
+    /// <param name="quickCompare">Set to true to skip the per directory comparison logic and
+    /// instead rely on the global platform comparison based on the currently running platform.</param>
     /// <returns></returns>
     [HttpGet("PathEndsWith/{*path}")]
-    public ActionResult<List<Series>> PathEndsWith([FromRoute] string path)
+    public ActionResult<List<Series>> PathEndsWith([FromRoute] string path, bool quickCompare = true)
     {
         var user = User;
         var query = path;
@@ -2845,9 +2848,14 @@ public ActionResult<List<Series>> PathEndsWith([FromRoute] string path)
         return RepoFactory.VideoLocalPlace.GetAll()
             .Where(a =>
             {
-                if (a.FullServerPath == null) return false;
+                if (a.FullServerPath == null)
+                    return false;
                 var dir = Path.GetDirectoryName(a.FullServerPath);
-                return dir != null && dir.EndsWith(query, StringComparison.OrdinalIgnoreCase);
+                if (string.IsNullOrEmpty(dir))
+                    return false;
+
+                var comparison = quickCompare ? Utils.PlatformComparison : Utils.GetComparisonFor(dir);
+                return dir != null && dir.EndsWith(query, comparison);
             })
             .SelectMany(a => a.VideoLocal?.AnimeEpisodes ?? Enumerable.Empty<SVR_AnimeEpisode>())
             .DistinctBy(a => a.AnimeSeriesID)

Shoko.Server/FileHelper/Hasher.cs

@@ -140,7 +140,7 @@ public static string GetVersion()
     public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgress, bool getCRC32, bool getMD5, bool getSHA1)
     {
         var rhash = new Hashes();
-        if (Finalise.ModuleHandle == IntPtr.Zero && !Utils.IsLinux)
+        if (Finalise.ModuleHandle == IntPtr.Zero && !Utils.IsLinuxOrMac)
             return CalculateHashes_here(strPath, onHashProgress, getCRC32, getMD5, getSHA1);
 
         var hash = new byte[56];
@@ -149,7 +149,7 @@ public static Hashes CalculateHashes(string strPath, OnHashProgress onHashProgre
         try
         {
             var filename = strPath;
-            if (!Utils.IsLinux)
+            if (!Utils.IsLinuxOrMac)
             {
                 filename = strPath.StartsWith(@"\\")
                     ? strPath

Shoko.Server/FileHelper/MD4Managed.cs

@@ -22,7 +22,7 @@ protected MD4()
     {
         var obj = CryptoConfig.CreateFromName(hashName);
         // in case machine.config isn't configured to use any MD4 implementation
-        if (obj == null || Utils.IsRunningOnLinuxOrMac())
+        if (obj == null || Utils.IsLinuxOrMac)
         {
             obj = new MD4Managed();
         }

Shoko.Server/Renamer/RenameFileService.cs

@@ -240,14 +240,15 @@ private static RelocationResult UnAbstractResult(SVR_VideoLocal_Place place, Abs
         var newRelativeDirectory = shouldMove && !result.SkipMove ? result.Path : Path.GetDirectoryName(place.FilePath);
         var newRelativePath = !string.IsNullOrEmpty(newRelativeDirectory) && newRelativeDirectory.Length > 0 ? Path.Combine(newRelativeDirectory, newFileName) : newFileName;
         var newFullPath = Path.Combine(newImportFolder.Path, newRelativePath);
+        var oldFullPath = place.FullServerPath;
+        var comparison = Utils.GetComparisonFor(Path.GetDirectoryName(newFullPath), Path.GetDirectoryName(oldFullPath));
         return new()
         {
             Success = true,
             ImportFolder = newImportFolder,
             RelativePath = newRelativePath,
-            // TODO: Handle file-systems that are or aren't case sensitive.
-            Renamed = !string.Equals(place.FileName, result.FileName, StringComparison.OrdinalIgnoreCase),
-            Moved = !string.Equals(Path.GetDirectoryName(place.FullServerPath), Path.GetDirectoryName(newFullPath), StringComparison.OrdinalIgnoreCase),
+            Renamed = !string.Equals(place.FileName, result.FileName, comparison),
+            Moved = !string.Equals(Path.GetDirectoryName(oldFullPath), Path.GetDirectoryName(newFullPath), comparison),
         };
     }
 

Shoko.Server/Repositories/Cached/ImportFolderRepository.cs

@@ -4,6 +4,7 @@
 using Shoko.Models.Server;
 using Shoko.Server.Databases;
 using Shoko.Server.Models;
+using Shoko.Server.Utilities;
 
 namespace Shoko.Server.Repositories.Cached;
 
@@ -73,10 +74,19 @@ public SVR_ImportFolder SaveImportFolder(ImportFolder folder)
             throw new Exception("Cannot find Import Folder location");
         }
 
-        if (GetAll().ExceptBy([folder.ImportFolderID], iF => iF.ImportFolderID).Any(iF => folder.ImportFolderLocation.StartsWith(iF.ImportFolderLocation, StringComparison.OrdinalIgnoreCase) || iF.ImportFolderLocation.StartsWith(folder.ImportFolderLocation, StringComparison.OrdinalIgnoreCase)))
+        if (GetAll().ExceptBy([folder.ImportFolderID], iF => iF.ImportFolderID).Any(iF =>
+        {
+            var comparison = Utils.GetComparisonFor(folder.ImportFolderLocation, iF.ImportFolderLocation);
+            var newLocation = folder.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (newLocation[^1] != Path.DirectorySeparatorChar)
+                newLocation += Path.DirectorySeparatorChar;
+            var existingLocation = iF.ImportFolderLocation.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar);
+            if (existingLocation[^1] != Path.DirectorySeparatorChar)
+                existingLocation += Path.DirectorySeparatorChar;
+            return newLocation.StartsWith(existingLocation, comparison) || existingLocation.StartsWith(newLocation, comparison);
+        }))
             throw new Exception("Unable to nest an import folder within another import folder.");
 
-
         ns.ImportFolderName = folder.ImportFolderName;
         ns.ImportFolderLocation = folder.ImportFolderLocation;
         ns.IsDropDestination = folder.IsDropDestination;

Shoko.Server/Scheduling/Jobs/Shoko/HashFileJob.cs

@@ -277,7 +277,7 @@ private bool HandleReadOnlyException(Exception ex)
             var info = new FileInfo(FilePath);
             if (info.IsReadOnly) info.IsReadOnly = false;
 
-            if (!info.IsReadOnly && !Utils.IsRunningOnLinuxOrMac())
+            if (!info.IsReadOnly && !Utils.IsLinuxOrMac)
             {
                 return true;
             }

Shoko.Server/Server/ShokoServer.cs

@@ -117,7 +117,7 @@ public bool StartUpServer()
 
     private bool CheckBlockedFiles()
     {
-        if (Utils.IsRunningOnLinuxOrMac()) return true;
+        if (Utils.IsLinuxOrMac) return true;
 
         var programlocation =
             Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);

Shoko.Server/Services/ActionService.cs

@@ -162,12 +162,14 @@ public async Task RunImport_ScanFolder(int importFolderID, bool skipMyList = fal
                 dictFilesExisting[vl.FullServerPath] = vl;
             }
 
-            Utils.GetFilesForImportFolder(folder.BaseDirectory, ref fileList);
+            var baseDirectory = folder.BaseDirectory;
+            var comparer = Utils.GetComparerFor(baseDirectory.FullName);
+            Utils.GetFilesForImportFolder(baseDirectory, ref fileList);
 
             // Get Ignored Files and remove them from the scan listing
             var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos().SelectMany(a => a.Places)
                 .Select(a => a.FullServerPath).Where(a => !string.IsNullOrEmpty(a)).ToList();
-            fileList = fileList.Except(ignoredFiles, StringComparer.InvariantCultureIgnoreCase).ToList();
+            fileList = fileList.Except(ignoredFiles, comparer).ToList();
 
             // get a list of all files in the share
             foreach (var fileName in fileList)
@@ -210,19 +212,27 @@ public async Task RunImport_DropFolders()
     {
         var settings = _settingsProvider.GetSettings();
         var scheduler = await _schedulerFactory.GetScheduler();
+        // Get Ignored Files and remove them from the scan listing
+        var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos()
+            .SelectMany(a => a.Places)
+            .Select(a => a.FullServerPath)
+            .Where(a => !string.IsNullOrEmpty(a))
+            .ToList();
+
         // get a complete list of files
         var fileList = new List<string>();
-        foreach (var share in RepoFactory.ImportFolder.GetAll())
+        foreach (var folder in RepoFactory.ImportFolder.GetAll())
         {
-            if (!share.FolderIsDropSource) continue;
-            Utils.GetFilesForImportFolder(share.BaseDirectory, ref fileList);
+            if (!folder.FolderIsDropSource) continue;
+
+            var fileListForFolder = new List<string>();
+            var baseDirectory = folder.BaseDirectory;
+            var comparer = Utils.GetComparerFor(baseDirectory.FullName);
+            Utils.GetFilesForImportFolder(baseDirectory, ref fileListForFolder);
+            fileListForFolder = fileListForFolder.Except(ignoredFiles, comparer).ToList();
+            fileList.AddRange(fileListForFolder);
         }
 
-        // Get Ignored Files and remove them from the scan listing
-        var ignoredFiles = RepoFactory.VideoLocal.GetIgnoredVideos().SelectMany(a => a.Places)
-            .Select(a => a.FullServerPath).Where(a => !string.IsNullOrEmpty(a)).ToList();
-        fileList = fileList.Except(ignoredFiles, StringComparer.InvariantCultureIgnoreCase).ToList();
-
         // get a list of all the shares we are looking at
         int filesFound = 0, videosFound = 0;
         var i = 0;