[WIP] Improve File.Copy by using the block copy operation for ReFS on Windows

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.DUPLICATE_EXTENTS_DATA.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ns-winioctl-duplicate_extents_data
+        internal struct DUPLICATE_EXTENTS_DATA
+        {
+            internal IntPtr FileHandle;
+            internal long SourceFileOffset;
+            internal long TargetFileOffset;
+            internal long ByteCount;
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.DeviceIoControl.cs

@@ -15,6 +15,18 @@ internal static partial class Kernel32
         // https://docs.microsoft.com/windows-hardware/drivers/ddi/ntddstor/ni-ntddstor-ioctl_storage_read_capacity
         internal const int IOCTL_STORAGE_READ_CAPACITY = 0x002D5140;
 
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ni-winioctl-fsctl_set_sparse
+        internal const int FSCTL_SET_SPARSE = 0x000900c4;
+
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ni-winioctl-fsctl_get_integrity_information
+        internal const int FSCTL_GET_INTEGRITY_INFORMATION = 0x0009027C;
+
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ni-winioctl-fsctl_set_integrity_information
+        internal const int FSCTL_SET_INTEGRITY_INFORMATION = 0x0009C280;
+
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ni-winioctl-fsctl_duplicate_extents_to_file
+        internal const int FSCTL_DUPLICATE_EXTENTS_TO_FILE = 0x00098344;
+
         [LibraryImport(Libraries.Kernel32, EntryPoint = "DeviceIoControl", SetLastError = true)]
         [return: MarshalAs(UnmanagedType.Bool)]
         internal static unsafe partial bool DeviceIoControl(

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FILE_DISPOSITION_INFO.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        // From FILE_INFO_BY_HANDLE_CLASS
+        // Use for SetFileInformationByHandle
+        internal const int FileDispositionInfo = 4;
+
+        internal struct FILE_DISPOSITION_INFO
+        {
+            internal BOOLEAN DeleteFile;
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FILE_SET_SPARSE_BUFFER.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ns-winioctl-file_set_sparse_buffer
+        internal struct FILE_SET_SPARSE_BUFFER
+        {
+            internal BOOLEAN SetSparse;
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FSCTL_GET_INTEGRITY_INFORMATION_BUFFER.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ns-winioctl-fsctl_get_integrity_information_buffer
+        internal struct FSCTL_GET_INTEGRITY_INFORMATION_BUFFER
+        {
+            internal ushort ChecksumAlgorithm;
+            internal ushort Reserved;
+            internal uint Flags;
+            internal uint ChecksumChunkSizeInBytes;
+            internal uint ClusterSizeInBytes;
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FSCTL_SET_INTEGRITY_INFORMATION_BUFFER.cs

@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        // https://learn.microsoft.com/windows/win32/api/winioctl/ns-winioctl-fsctl_set_integrity_information_buffer
+        internal struct FSCTL_SET_INTEGRITY_INFORMATION_BUFFER
+        {
+            internal ushort ChecksumAlgorithm;
+            internal ushort Reserved;
+            internal uint Flags;
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileAttributes.cs

@@ -10,6 +10,7 @@ internal static partial class FileAttributes
             internal const int FILE_ATTRIBUTE_NORMAL = 0x00000080;
             internal const int FILE_ATTRIBUTE_READONLY = 0x00000001;
             internal const int FILE_ATTRIBUTE_DIRECTORY = 0x00000010;
+            internal const int FILE_ATTRIBUTE_SPARSE_FILE = 0x00000200;
             internal const int FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400;
         }
     }

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetDiskFreeSpace.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        [LibraryImport(Libraries.Kernel32, EntryPoint = "GetDiskFreeSpaceW",  SetLastError = true, StringMarshalling = StringMarshalling.Utf16)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static partial bool GetDiskFreeSpace(string drive, out int sectorsPerCluster, out int bytesPerSector, out int numberOfFreeClusters, out int totalNumberOfClusters);
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetVolumeInformation.cs

@@ -21,5 +21,6 @@ internal static unsafe partial bool GetVolumeInformation(
             int fileSystemNameBufLen);
 
         internal const uint FILE_SUPPORTS_ENCRYPTION = 0x00020000;
+        internal const uint FILE_SUPPORTS_BLOCK_REFCOUNTING = 0x08000000;
     }
 }

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetVolumeInformationByHandle.cs

@@ -0,0 +1,27 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+using Microsoft.Win32.SafeHandles;
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        [LibraryImport(Libraries.Kernel32, EntryPoint = "GetVolumeInformationByHandleW", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static unsafe partial bool GetVolumeInformationByHandle(
+            SafeFileHandle hFile,
+            char* volumeName,
+            uint volumeNameBufLen,
+            uint* volSerialNumber,
+            uint* maxFileNameLen,
+            uint* fileSystemFlags,
+            char* fileSystemName,
+            uint fileSystemNameBufLen);
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetVolumePathName.cs

@@ -0,0 +1,48 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        [LibraryImport(Libraries.Kernel32, EntryPoint = "GetVolumePathNameW", SetLastError = true, StringMarshalling = StringMarshalling.Utf16)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        private static unsafe partial bool GetVolumePathName(char* lpszFileName, char* lpszVolumePathName, int cchBufferLength);
+
+        public static unsafe string GetVolumePathName(string fileName)
+        {
+            // Ensure we have the prefix
+            fileName = PathInternal.EnsureExtendedPrefixIfNeeded(fileName);
+
+            // Ensure our output buffer will be long enough (see https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getvolumepathnamew#remarks)
+            ValueStringBuilder vsb = new(stackalloc char[Interop.Kernel32.MAX_PATH]); //note: MAX_PATH is not a hard limit, but would only be exceeded by a long path
+            PathHelper.GetFullPathName(fileName, ref vsb);
+
+            // Call the actual API
+            fixed (char* lpszFileName = fileName)
+            {
+                fixed (char* lpszVolumePathName = vsb.RawChars)
+                {
+                    // + 1 because \0 is not included in Length from GetFullPathName, but should exist
+                    Debug.Assert(vsb.Length + 1 <= vsb.Capacity);
+                    if (GetVolumePathName(lpszFileName, lpszVolumePathName, vsb.Length + 1))
+                    {
+                        vsb.Length = vsb.RawChars.IndexOf('\0');
+                        return vsb.ToString();
+                    }
+                }
+            }
+
+            // Deal with error
+            int error = Marshal.GetLastWin32Error();
+            Debug.Assert(error != Interop.Errors.ERROR_INSUFFICIENT_BUFFER);
+            throw Win32Marshal.GetExceptionForWin32Error(error, fileName);
+        }
+    }
+}

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.SetEndOfFile.cs

@@ -0,0 +1,15 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Win32.SafeHandles;
+using System.Runtime.InteropServices;
+
+internal static partial class Interop
+{
+    internal static partial class Kernel32
+    {
+        [LibraryImport(Libraries.Kernel32, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        internal static partial bool SetEndOfFile(SafeFileHandle hFile);
+    }
+}

src/libraries/System.IO.FileSystem/tests/File/Copy.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using Xunit;
 
@@ -418,5 +419,80 @@ public void EnsureThrowWhenCopyToNonSharedFile()
             using var stream = new FileStream(file1, FileMode.Open, FileAccess.Read, FileShare.None);
             Assert.Throws<IOException>(() => File.Copy(file2, file1, overwrite: true));
         }
+
+        [ConditionalTheory(typeof(MountHelper), nameof(MountHelper.CanCreateSymbolicLinks)),
+            InlineData("", ""),
+            /*InlineData(":a", ""),*/
+            InlineData("", ":a")/*,
+            InlineData(":a", ":a")*/]
+        //todo: is copying from an ADS meant to fail?
+        [PlatformSpecific(TestPlatforms.Windows)]
+        public void WindowsAlternateDataStreamSymlinkTest(string stream1, string stream2)
+        {
+            // This test checks copying all combinations of alternate data streams with all combinations of symlinks referencing them.
+            // This test exists to check we don't cause a BSOD when using ReFS block copy operation on alternative data streams (pending? rolled out fix from Windows team), and that it has the correct behaviour.
+
+            string sourceFile = GetTestFilePath();
+            string destFile = GetTestFilePath();
+
+            void Test(string src, string dst)
+            {
+                try
+                {
+                    File.WriteAllText(sourceFile, "abc");
+                    File.WriteAllText(destFile, "def");
+                    File.WriteAllText(sourceFile + stream1, "ghi");
+                    File.WriteAllText(destFile + stream2, "jkl");
+
+                    File.Copy(src, dst, true);
+
+                    if (stream1 != "") Assert.Equal("abc", File.ReadAllText(sourceFile));
+                    if (stream2 != "") Assert.Equal("def", File.ReadAllText(destFile));
+                    Assert.Equal("ghi", File.ReadAllText(sourceFile + stream1));
+                    Assert.Equal("ghi", File.ReadAllText(destFile + stream2));
+                }
+                catch (Exception ex)
+                {
+                    throw new Exception($"Failed with src={src}, dst={dst}.", ex);
+                }
+            }
+
+            File.CreateSymbolicLink(sourceFile + ".link", sourceFile + stream1);
+            File.CreateSymbolicLink(destFile + ".link", destFile + stream2);
+
+            Test(sourceFile + stream1, destFile + stream2);
+            Test(sourceFile + stream1, destFile + ".link");
+            Test(sourceFile + ".link", destFile + stream2);
+            Test(sourceFile + ".link", destFile + ".link");
+        }
+
+        [Fact]
+        [PlatformSpecific(TestPlatforms.Windows)]
+        public unsafe void WindowsCheckSparseness()
+        {
+            string sourceFile = GetTestFilePath();
+            string destFile = GetTestFilePath();
+
+            File.WriteAllText(sourceFile, "abc");
+            File.WriteAllText(destFile, "def");
+
+            Assert.True((File.GetAttributes(sourceFile) & FileAttributes.SparseFile) == 0);
+            File.Copy(sourceFile, destFile, true);
+            Assert.True((File.GetAttributes(destFile) & FileAttributes.SparseFile) == 0);
+            Assert.Equal("abc", File.ReadAllText(sourceFile));
+
+            using (FileStream file = File.Open(sourceFile, FileMode.Open))
+            {
+                Assert.True(Interop.Kernel32.DeviceIoControl(file.SafeFileHandle, Interop.Kernel32.FSCTL_SET_SPARSE, null, 0, null, 0, out _, 0));
+            }
+            File.WriteAllText(destFile, "def");
+
+            Assert.True((File.GetAttributes(sourceFile) & FileAttributes.SparseFile) != 0);
+            File.Copy(sourceFile, destFile, true);
+            Assert.True((File.GetAttributes(destFile) & FileAttributes.SparseFile) != 0);
+            Assert.Equal("abc", File.ReadAllText(sourceFile));
+        }
+
+        // Todo: add a way to run all these on ReFS, and a test to check we actually cloned the reference, not just the data on ReFS.
     }
 }

src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj

@@ -226,6 +226,10 @@
     <Compile Include="FileInfo\AppendText.cs" />
     <Compile Include="FileInfo\CopyTo.cs" />
     <!-- Helpers -->
+    <Compile Include="$(CommonPath)Interop\Windows\Interop.BOOLEAN.cs" Link="Interop\Windows\Interop.BOOLEAN.cs" />
+    <Compile Include="$(CommonPath)Interop\Windows\Interop.Libraries.cs" Link="Interop\Windows\Interop.Libraries.cs" />
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.DeviceIoControl.cs" Link="Interop\Windows\Interop.DeviceIoControl.cs" />
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FILE_SET_SPARSE_BUFFER.cs" Link="Interop\Windows\Interop.FILE_SET_SPARSE_BUFFER.cs" />
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GenericOperations.cs" Link="Interop\Windows\Interop.GenericOperations.cs" />
     <Compile Include="$(CommonTestPath)System\Buffers\NativeMemoryManager.cs" Link="Common\System\Buffers\NativeMemoryManager.cs" />
     <Compile Include="$(CommonTestPath)System\IO\TempFile.cs" Link="Common\System\IO\TempFile.cs" />

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -1657,6 +1657,18 @@
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.DeviceIoControl.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.DeviceIoControl.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FILE_SET_SPARSE_BUFFER.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.FILE_SET_SPARSE_BUFFER.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FSCTL_GET_INTEGRITY_INFORMATION_BUFFER.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.FSCTL_GET_INTEGRITY_INFORMATION_BUFFER.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FSCTL_SET_INTEGRITY_INFORMATION_BUFFER.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.FSCTL_SET_INTEGRITY_INFORMATION_BUFFER.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.DUPLICATE_EXTENTS_DATA.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.DUPLICATE_EXTENTS_DATA.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.ExpandEnvironmentStrings.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.ExpandEnvironmentStrings.cs</Link>
     </Compile>
@@ -1666,6 +1678,9 @@
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FILE_ALLOCATION_INFO.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.FILE_ALLOCATION_INFO.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FILE_DISPOSITION_INFO.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.FILE_DISPOSITION_INFO.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.FILE_END_OF_FILE_INFO.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.FILE_END_OF_FILE_INFO.cs</Link>
     </Compile>
@@ -1732,6 +1747,9 @@
     <Compile Include="$(CommonPath)\Interop\Windows\Kernel32\Interop.GetCurrentProcessId.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.GetCurrentProcessId.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GetDiskFreeSpace.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.GetDiskFreeSpace.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)\Interop\Windows\Kernel32\Interop.GetCurrentThreadId.cs">
       <Link>Interop\Windows\Kernel32\Interop.GetCurrentThreadId.cs</Link>
     </Compile>
@@ -1792,6 +1810,12 @@
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GetVolumeInformation.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.GetVolumeInformation.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GetVolumeInformationByHandle.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.GetVolumeInformationByHandle.cs</Link>
+    </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GetVolumePathName.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.GetVolumePathName.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.GlobalMemoryStatusEx.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.GlobalMemoryStatusEx.cs</Link>
     </Compile>
@@ -1894,6 +1918,9 @@
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.SetCurrentDirectory.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.SetCurrentDirectory.cs</Link>
     </Compile>
+    <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.SetEndOfFile.cs">
+      <Link>Common\Interop\Windows\Kernel32\Interop.SetEndOfFile.cs</Link>
+    </Compile>
     <Compile Include="$(CommonPath)Interop\Windows\Kernel32\Interop.SetFileAttributes.cs">
       <Link>Common\Interop\Windows\Kernel32\Interop.SetFileAttributes.cs</Link>
     </Compile>