MemoryMappedFile 内存映射+ Parallel 并行分块读写大文件

最新推荐文章于 2024-01-19 07:14:48 发布

@朱较瘦

最新推荐文章于 2024-01-19 07:14:48 发布

阅读量6.3k

点赞数 2

分类专栏： C#

本文链接：https://blog.csdn.net/zhulongxi/article/details/51899751

版权

C# 专栏收录该内容

67 篇文章 2 订阅

订阅专栏

 private static void SpiltFile(string srcFile, int portionSize)
        {
            string savedPath = @"\\stcsrv-c81\MMFeedHealthyDatacache\2016_07_10\Feedkeys\No_Process_test.txt";
            FileInfo fi = new FileInfo(srcFile);
            // total size in bytes
            Int64 size = fi.Length;
            object locker = new object();
            object writeLock = new object();
            List<MappedFile> mappedFiles = new List<MappedFile>();
            Int64 fileToRead = size;//文件总的大小
      
            portionSize = portionSize * 1024 * 1024; //每块大小

            Int64 portion = (Int64)Math.Ceiling(size * 1.0 / portionSize); //分成多少块

            Int64 fileOffset = 0;

            MemoryMappedViewAccessor mmf_reader = null;
            Stopwatch watch = Stopwatch.StartNew();
            watch.Start();
            Int64 fileSize = 0;
            using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(srcFile, FileMode.Open, "xyz", size))
            {
                //using (var writeMap = MemoryMappedFile.CreateFromFile(savedPath, FileMode.Create, "test", size, MemoryMappedFileAccess.ReadWrite))
                //{
                    //bool mutexCreated;
                    //Mutex mutex = new Mutex(true, "testmapmutex", out mutexCreated);//进程间同步
                    Parallel.For(0, portion, (i, ParallelLoopState) =>
                    {

                        //for (int i = 26; i < portion; i++)
                        //{
                        lock (locker)
                        {
                            fileSize = Math.Min(portionSize, fileToRead - portionSize * i);
                            if (fileSize > 0)
                            {
                                byte[] buffer;
                                using (mmf_reader = mmf.CreateViewAccessor(i * portionSize, fileSize, MemoryMappedFileAccess.Read))
                                {
                                    buffer = new byte[fileSize];
                                    mmf_reader.ReadArray(0, buffer, 0, (int)fileSize);
                                    mappedFiles.Add(new MappedFile
                                    {
                                        Offset = i * portionSize, //fileOffset,
                                        Buffer = buffer,
                                        FileSize = fileSize
                                    });
                                }

                                //fileToRead -= fileSize;
                                //lock (writeLock)
                                //{
                                //using (var writeMmf = MemoryMappedFile.OpenExisting("xyz"))
                                //{
                                //    using (var writeAccessor = writeMmf.CreateViewStream(i * portionSize, fileSize))
                                //    {
                                //        var w = new BinaryWriter(new FileStream(savedPath, FileMode.Create, FileAccess.Write));
                                //        //writeAccessor.WriteArray(i * portionSize, buffer, 0, buffer.Length);
                                //        //writeAccessor.Write(buffer, 0, buffer.Length);
                                //        w.Write(buffer);
                                //    }
                                //}

                                //using (MemoryMappedViewAccessor writeView = writeMap.CreateViewAccessor())
                                //{
                                //    writeView.WriteArray(i * portionSize, buffer, 0, (int)fileSize);
                                //}

                            }
                            //}
                        }

                    });
                }


            using (var writeMap = MemoryMappedFile.CreateFromFile(savedPath, FileMode.Create, "test", size, MemoryMappedFileAccess.ReadWrite))
            {
                using (MemoryMappedViewAccessor writeView = writeMap.CreateViewAccessor())
                {
                    Parallel.For(0, mappedFiles.Count, i =>
                    {
                        try
                        {
                            Monitor.Enter(locker);
                            writeView.WriteArray(mappedFiles[i].Offset, mappedFiles[i].Buffer, 0, (int)mappedFiles[i].FileSize);
                        }
                        catch (Exception)
                        {

                            throw;
                        }
                        finally
                        {
                            Monitor.Exit(locker);
                        }

                    });
                }
            }
            watch.Stop();
            Console.WriteLine(watch.ElapsedMilliseconds);
            #region MyRegion
        }

    public class MappedFile
    {
        public long Offset { get; set; }
        public byte[] Buffer { get; set; }
        public long FileSize { get; set; }
    }