如何在大型稀疏文件上创建快速有效的文件流写入

我有一个应用程序，可以在多个段中写入大文件。 我使用FileStream.Seek来定位每个wirte。 看来，当我在稀疏文件中的深位置调用FileStream.Write时，write会在所有前面的字节上触发“回填”操作（写入0），这很慢。

有没有更有效的方法来处理这种情况？

以下代码演示了该问题。 初始写入在我的机器上大约需要370 MS。

public void WriteToStream() { DateTime dt; using (FileStream fs = File.Create("C:\\testfile.file")) { fs.SetLength(1024 * 1024 * 100); fs.Seek(-1, SeekOrigin.End); dt = DateTime.Now; fs.WriteByte(255); } Console.WriteLine(@"WRITE MS: " + DateTime.Now.Subtract(dt).TotalMilliseconds.ToString()); } 

Answer 1

NTFS确实支持稀疏文件 ，但是没有p /调用一些本机方法就无法在.net中执行。

将文件标记为稀疏文件并不是很难，只要知道一旦文件被标记为稀疏文件，它就永远不能转换回非稀疏文件，除非将整个文件复制到新的非稀疏文件中。

示例用法

 class Program { [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, IntPtr InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped ); static void MarkAsSparseFile(SafeFileHandle fileHandle) { int bytesReturned = 0; NativeOverlapped lpOverlapped = new NativeOverlapped(); bool result = DeviceIoControl( fileHandle, 590020, //FSCTL_SET_SPARSE, IntPtr.Zero, 0, IntPtr.Zero, 0, ref bytesReturned, ref lpOverlapped); if(result == false) throw new Win32Exception(); } static void Main() { //Use stopwatch when benchmarking, not DateTime Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); using (FileStream fs = File.Create(@"e:\Test\test.dat")) { MarkAsSparseFile(fs.SafeFileHandle); fs.SetLength(1024 * 1024 * 100); fs.Seek(-1, SeekOrigin.End); fs.WriteByte(255); } stopwatch.Stop(); //Returns 2 for sparse files and 1127 for non sparse Console.WriteLine(@"WRITE MS: " + stopwatch.ElapsedMilliseconds); } } 

---

一旦文件被标记为稀疏文件，它现在的行为就像你排除了它在评论中的行为一样。 您不需要编写一个字节来将文件标记为设置大小。

 static void Main() { string filename = @"e:\Test\test.dat"; using (FileStream fs = new FileStream(filename, FileMode.Create)) { MarkAsSparseFile(fs.SafeFileHandle); fs.SetLength(1024 * 1024 * 25); } } 

Answer 2

以下是使用稀疏文件的一些代码：

 using System; using System.ComponentModel; using System.IO; using System.Runtime.InteropServices; using System.Text; using System.Threading; using Microsoft.Win32.SafeHandles; public static class SparseFiles { private const int FILE_SUPPORTS_SPARSE_FILES = 64; private const int FSCTL_SET_SPARSE = 0x000900c4; private const int FSCTL_SET_ZERO_DATA = 0x000980c8; public static void MakeSparse(this FileStream fileStream) { var bytesReturned = 0; var lpOverlapped = new NativeOverlapped(); var result = DeviceIoControl( fileStream.SafeFileHandle, FSCTL_SET_SPARSE, IntPtr.Zero, 0, IntPtr.Zero, 0, ref bytesReturned, ref lpOverlapped); if (!result) { throw new Win32Exception(); } } public static void SetSparseRange(this FileStream fileStream, long fileOffset, long length) { var fzd = new FILE_ZERO_DATA_INFORMATION(); fzd.FileOffset = fileOffset; fzd.BeyondFinalZero = fileOffset + length; var lpOverlapped = new NativeOverlapped(); var dwTemp = 0; var result = DeviceIoControl( fileStream.SafeFileHandle, FSCTL_SET_ZERO_DATA, ref fzd, Marshal.SizeOf(typeof(FILE_ZERO_DATA_INFORMATION)), IntPtr.Zero, 0, ref dwTemp, ref lpOverlapped); if (!result) { throw new Win32Exception(); } } public static bool SupportedOnVolume(string filename) { var targetVolume = Path.GetPathRoot(filename); var fileSystemName = new StringBuilder(300); var volumeName = new StringBuilder(300); uint lpFileSystemFlags; uint lpVolumeSerialNumber; uint lpMaxComponentLength; var result = GetVolumeInformationW( targetVolume, volumeName, (uint)volumeName.Capacity, out lpVolumeSerialNumber, out lpMaxComponentLength, out lpFileSystemFlags, fileSystemName, (uint)fileSystemName.Capacity); if (!result) { throw new Win32Exception(); } return (lpFileSystemFlags & FILE_SUPPORTS_SPARSE_FILES) == FILE_SUPPORTS_SPARSE_FILES; } [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, IntPtr InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped); [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool DeviceIoControl( SafeFileHandle hDevice, int dwIoControlCode, ref FILE_ZERO_DATA_INFORMATION InBuffer, int nInBufferSize, IntPtr OutBuffer, int nOutBufferSize, ref int pBytesReturned, [In] ref NativeOverlapped lpOverlapped); [DllImport("kernel32.dll", EntryPoint = "GetVolumeInformationW")] [return: MarshalAs(UnmanagedType.Bool)] private static extern bool GetVolumeInformationW( [In] [MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName, [Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpVolumeNameBuffer, uint nVolumeNameSize, out uint lpVolumeSerialNumber, out uint lpMaximumComponentLength, out uint lpFileSystemFlags, [Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpFileSystemNameBuffer, uint nFileSystemNameSize); [StructLayout(LayoutKind.Sequential)] private struct FILE_ZERO_DATA_INFORMATION { public long FileOffset; public long BeyondFinalZero; } } 

并用示例代码来测试上面的类。

 class Program { static void Main(string[] args) { using (var fileStream = new FileStream("test", FileMode.Create, FileAccess.ReadWrite, FileShare.None)) { fileStream.SetLength(1024 * 1024 * 128); fileStream.MakeSparse(); fileStream.SetSparseRange(0, fileStream.Length); } } } 

希望这可以帮助