C# 读写快速读写大文件

需要开发一个客户端来存储一些期货数据.

原本是用sqlite来进行存储和读写,但是发现sqlite的效率实在是差.

也许是代码问题吧,就想到用文本的方法来存储.

文件通过按天,按种类存储,一个是data,一个是index.

一条数据,分别写入到这2个文件里面.data写入全数据,index记录时间和data里面的偏移,以便查找.

需要解决的问题:

1.因为行情是文本的,直接存储会占用大量的空间,如何节约空间存储.

2.因为按天存储数据量也是比较大的,按照一天6个小时交易时间,一秒2条数据来算,索引文件也是比较大的.如何按照时间快速查找数据也是函待解决的问题.

------------------------------------------------------------------------

解决思路:

1.压缩存储,C# 带有7z或者zip 的开源库

2.多条存储,一分钟写一次文件库,索引按照分钟来进行.对一分钟内的数据进行压缩,由于行情数据比较相似,所以压缩率会高,

3.查找时候,提供2种方法,一种是把索引文件读入内存进行查找,一种是打开索引文件逐条直接搜索.

------------------------------------------------------------------------

代码:

写:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using ICSharpCode.SharpZipLib.Zip.Compression;
namespace WriteFileTest
{
    class Program
    {
        public struct Tick
        {
            public string Date;
            public string Type;
            public string DataBody;
        }
        public static Dictionary<string, string> CompressDic = new Dictionary<string, string>();
        public static List<Tick> tmpTickList = new List<Tick>();
        static void Main(string[] args)
        {
            DateTime d = DateTime.Now;
            string tmpTime = string.Empty;
            while (true)
            {
                Tick new_tick = new Tick();
                new_tick.Type = sGetType();
                d = GetNextTime(d);
                if (tmpTime == "")
                {
                    tmpTime = d.ToString("yyyyMMddHHmm");
                }
                new_tick.Date = d.ToString("yyyyMMddHHmmssfff");
                new_tick.DataBody = new_tick.Type + new_tick.Date + DateTime.Now.ToString();
                //Console.WriteLine(d.ToString("yyyyMMddHHmmssfff") + "   " + new_tick.Type);
                //判断是否切换分钟,如果切换,入库一次
                if (d.ToString("yyyyMMddHHmm") != tmpTime)
                {
                    if (!WriteFile(tmpTickList, tmpTime))
                    {
                        Console.WriteLine("写文件发生错误!");
                    }
                    else
                    {
                        tmpTime = d.ToString("yyyyMMddHHmm");
                        tmpTickList.Clear();
                        CompressDic.Clear();
                    }
                }
                tmpTickList.Add(new_tick);

            }
        }

        /// <summary>
        /// 写文件
        /// </summary>
        /// <param name="sTick">传入的需要写文件的数据</param>
        public static bool WriteFile(List<Tick> sListTick,string Time)
        {

            foreach (Tick t in sListTick)
            {
                if (!CompressDic.ContainsKey(t.Type))
                {
                    CompressDic.Add(t.Type, "");
                }
                //todo:把文件推入压缩队列
                CompressDic[t.Type] = CompressDic[t.Type] + t.Type + "|" + t.Date + "|" + t.DataBody + "|" + "\r";
            }
            foreach (KeyValuePair<string, string> kvp in CompressDic)
            {
                WriteFile(kvp.Key + "_" + Time.Substring(0, 8), Time, kvp.Value);
            }
            //Console.ReadLine();
            return true;
        }

        public static void WriteFile(string FileName, string FileIndex, string FileBody)
        {
            Console.WriteLine("------------------------------------------------------");
            Console.WriteLine("文件名称:" + FileName);
            Console.WriteLine("文件索引:" + FileIndex);
            Console.WriteLine("文件内容:" + FileBody);

            FileStream fs = new FileStream("data/"+FileName+".data", FileMode.Append);
            FileStream fs1 = new FileStream("data/" + FileName + ".index", FileMode.Append);
            //获得字节数组
            byte[] data = CompressByte(Encoding.UTF8.GetBytes(FileBody));
            byte[] return_key = new UTF8Encoding().GetBytes("\r");
            byte[] data1 = new UTF8Encoding().GetBytes(FileIndex +"|"+ data.Length.ToString());
            //开始写入
            fs.Write(data, 0, data.Length);
            fs1.Write(data1, 0, data1.Length);
            fs1.Write(return_key, 0, return_key.Length);
            //清空缓冲区、关闭流
            fs.Flush();
            fs.Close();
            fs1.Flush();
            fs1.Close();
            return;
        }

        /// <summary>
        /// 随机获得品种
        /// </summary>
        /// <returns></returns>
        public static string sGetType()
        {
            Random r = new Random();
            int k = r.Next(0, 2);
            switch (k)
            {
                case 0:
                    return "CU";
                case 1:
                    return "IF";
                default:
                    return "CU";
            }
        }

        /// <summary>
        /// 随机下一个毫秒时间
        /// </summary>
        /// <param name="d"></param>
        /// <returns></returns>
        public static DateTime GetNextTime(DateTime d)
        {
            d = d.AddMilliseconds(500);
            return d;
        }


        public static byte[] CompressByte(byte[] byteCache)
        {
            //return byteCache;
            //Console.WriteLine(Encoding.UTF8.GetString(byteCache));
            Deflater deFlater = new Deflater(Deflater.BEST_COMPRESSION);//实例化压缩对象
            deFlater.SetInput(byteCache);//设定压缩目标
            deFlater.Finish();//标识已完成对压缩目标的设置
            MemoryStream mStream = new MemoryStream();//内存流
            byte[] comCache = new byte[byteCache.Length];//压缩缓存
            while (!deFlater.IsFinished)
            {
                int comLength = deFlater.Deflate(comCache);//压缩（通过SetInput）指定字节数组，并将压缩结果写入缓存
                mStream.Write(comCache, 0, comLength);//把压缩缓存写入内存流
            }
            byte[] comResult = mStream.ToArray();//从内存流中把压缩结果转换回字节数组
            mStream.Close();//关闭内存流
            return comResult;//返回压缩结果
        }

        //public static byte[] DeCompressByte(byte[] byteCache)
        //{
        //    Inflater inflater = new Inflater();//创建解压实例
        //    inflater.SetInput(byteCache);//设置需要解压的字节
        //    MemoryStream mStream = new MemoryStream();//创建内存流实例
        //    byte[] inflaterCache = new byte[9999999999];//创建解压缓存字节数据组
        //    while (!inflater.IsFinished)//开始解压
        //    {
        //        int inflaterLength = inflater.Inflate(inflaterCache);//解压写入缓存数组
        //        mStream.Write(inflaterCache, 0, inflaterLength);//解压后的缓存数据写入内存流
        //    }
        //    byte[] deComResult = mStream.ToArray();//解压结果
        //    mStream.Close();//关闭内容流服务
        //    return deComResult;//返回解压的后的字节
        //}

    }






}

读:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression;
using System.IO;

namespace ReadFileTest
{
    class Program
    {

        public struct Tick
        {
            public string s_Date;
            public int i_length;
            public int i_Offset;
        }

        public static Dictionary<string, Tick> DataDic = new Dictionary<string, Tick>();

        static void Main(string[] args)
        {
            GetFileInfo("201408151846", "CU");
            Console.ReadLine();
        }

        public static void GetFileInfo(string s_Time, string s_Type)
        {
            string FileDataName = s_Type + "_" + s_Time.Substring(0, 8) + "." + "data";
            string FileIndexName = s_Type + "_" + s_Time.Substring(0, 8) + "." + "index";
            Console.WriteLine(FileDataName);
            Console.WriteLine(FileIndexName);
            int i = 0;
            int i_Offset = 0;
            using (StreamReader sr = new StreamReader(@"E:\Documents\Visual Studio 2010\Projects\WriteFileTest\WriteFileTest\bin\Debug\data\" + FileIndexName))
            {
                String line;
                while ((line = sr.ReadLine()) != null)
                {
                    Tick getTick = new Tick();
                    string s1 = line.Substring(0, line.IndexOf("|"));
                    string s2 = line.Substring(line.IndexOf("|") + 1);
                    Console.WriteLine("第" + i + "条的时间是:" + s1 + " 长度是:" + s2 + "\t\t偏移:" + i_Offset.ToString());
                    getTick.i_length = Int32.Parse(s2);
                    getTick.s_Date = s1;
                    getTick.i_Offset = i_Offset;
                    i_Offset = i_Offset + Int32.Parse(s2);
                    DataDic.Add("CU" + s1, getTick);
                    getData(FileDataName, getTick.i_length, getTick.i_Offset);
                    i++;
                }
                Console.WriteLine(i.ToString());
            }
            return;
        }


        public static string getData(string FileName, int i_length, long l_Offset)
        {
            string s_FileName = @"E:\Documents\Visual Studio 2010\Projects\WriteFileTest\WriteFileTest\bin\Debug\data\" + FileName;
            FileStream fs = new FileStream(s_FileName, FileMode.Open, FileAccess.Read);
            byte[] data= new byte[i_length];
            fs.Seek(l_Offset, SeekOrigin.Begin);
            fs.Read(data, 0, data.Length);
            string s=Encoding.UTF8.GetString(DeCompressByte(data));
            fs.Close();
            return s;
        }





        public static byte[] DeCompressByte(byte[] byteCache)
        {
            //return byteCache;
            Inflater inflater = new Inflater();//创建解压实例
            inflater.SetInput(byteCache);//设置需要解压的字节
            MemoryStream mStream = new MemoryStream();//创建内存流实例
            byte[] inflaterCache = new byte[99999];//创建解压缓存字节数据组
            while (!inflater.IsFinished)//开始解压
            {
                int inflaterLength = inflater.Inflate(inflaterCache);//解压写入缓存数组
                mStream.Write(inflaterCache, 0, inflaterLength);//解压后的缓存数据写入内存流
            }
            byte[] deComResult = mStream.ToArray();//解压结果
            mStream.Close();//关闭内容流服务
            return deComResult;//返回解压的后的字节
        }
    }
}