需要开发一个客户端来存储一些期货数据.
原本是用sqlite来进行存储和读写,但是发现sqlite的效率实在是差.
也许是代码问题吧,就想到用文本的方法来存储.
文件通过按天,按种类存储,一个是data,一个是index.
一条数据,分别写入到这2个文件里面.data写入全数据,index记录时间和data里面的偏移,以便查找.
需要解决的问题:
1.因为行情是文本的,直接存储会占用大量的空间,如何节约空间存储.
2.因为按天存储数据量也是比较大的,按照一天6个小时交易时间,一秒2条数据来算,索引文件也是比较大的.如何按照时间快速查找数据也是函待解决的问题.
------------------------------------------------------------------------
解决思路:
1.压缩存储,C# 带有7z或者zip 的开源库
2.多条存储,一分钟写一次文件库,索引按照分钟来进行.对一分钟内的数据进行压缩,由于行情数据比较相似,所以压缩率会高,
3.查找时候,提供2种方法,一种是把索引文件读入内存进行查找,一种是打开索引文件逐条直接搜索.
------------------------------------------------------------------------
代码:
写:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using ICSharpCode.SharpZipLib.Zip.Compression;
namespace WriteFileTest
{
class Program
{
public struct Tick
{
public string Date;
public string Type;
public string DataBody;
}
public static Dictionary<string, string> CompressDic = new Dictionary<string, string>();
public static List<Tick> tmpTickList = new List<Tick>();
static void Main(string[] args)
{
DateTime d = DateTime.Now;
string tmpTime = string.Empty;
while (true)
{
Tick new_tick = new Tick();
new_tick.Type = sGetType();
d = GetNextTime(d);
if (tmpTime == "")
{
tmpTime = d.ToString("yyyyMMddHHmm");
}
new_tick.Date = d.ToString("yyyyMMddHHmmssfff");
new_tick.DataBody = new_tick.Type + new_tick.Date + DateTime.Now.ToString();
//Console.WriteLine(d.ToString("yyyyMMddHHmmssfff") + " " + new_tick.Type);
//判断是否切换分钟,如果切换,入库一次
if (d.ToString("yyyyMMddHHmm") != tmpTime)
{
if (!WriteFile(tmpTickList, tmpTime))
{
Console.WriteLine("写文件发生错误!");
}
else
{
tmpTime = d.ToString("yyyyMMddHHmm");
tmpTickList.Clear();
CompressDic.Clear();
}
}
tmpTickList.Add(new_tick);
}
}
/// <summary>
/// 写文件
/// </summary>
/// <param name="sTick">传入的需要写文件的数据</param>
public static bool WriteFile(List<Tick> sListTick,string Time)
{
foreach (Tick t in sListTick)
{
if (!CompressDic.ContainsKey(t.Type))
{
CompressDic.Add(t.Type, "");
}
//todo:把文件推入压缩队列
CompressDic[t.Type] = CompressDic[t.Type] + t.Type + "|" + t.Date + "|" + t.DataBody + "|" + "\r";
}
foreach (KeyValuePair<string, string> kvp in CompressDic)
{
WriteFile(kvp.Key + "_" + Time.Substring(0, 8), Time, kvp.Value);
}
//Console.ReadLine();
return true;
}
public static void WriteFile(string FileName, string FileIndex, string FileBody)
{
Console.WriteLine("------------------------------------------------------");
Console.WriteLine("文件名称:" + FileName);
Console.WriteLine("文件索引:" + FileIndex);
Console.WriteLine("文件内容:" + FileBody);
FileStream fs = new FileStream("data/"+FileName+".data", FileMode.Append);
FileStream fs1 = new FileStream("data/" + FileName + ".index", FileMode.Append);
//获得字节数组
byte[] data = CompressByte(Encoding.UTF8.GetBytes(FileBody));
byte[] return_key = new UTF8Encoding().GetBytes("\r");
byte[] data1 = new UTF8Encoding().GetBytes(FileIndex +"|"+ data.Length.ToString());
//开始写入
fs.Write(data, 0, data.Length);
fs1.Write(data1, 0, data1.Length);
fs1.Write(return_key, 0, return_key.Length);
//清空缓冲区、关闭流
fs.Flush();
fs.Close();
fs1.Flush();
fs1.Close();
return;
}
/// <summary>
/// 随机获得品种
/// </summary>
/// <returns></returns>
public static string sGetType()
{
Random r = new Random();
int k = r.Next(0, 2);
switch (k)
{
case 0:
return "CU";
case 1:
return "IF";
default:
return "CU";
}
}
/// <summary>
/// 随机下一个毫秒时间
/// </summary>
/// <param name="d"></param>
/// <returns></returns>
public static DateTime GetNextTime(DateTime d)
{
d = d.AddMilliseconds(500);
return d;
}
public static byte[] CompressByte(byte[] byteCache)
{
//return byteCache;
//Console.WriteLine(Encoding.UTF8.GetString(byteCache));
Deflater deFlater = new Deflater(Deflater.BEST_COMPRESSION);//实例化压缩对象
deFlater.SetInput(byteCache);//设定压缩目标
deFlater.Finish();//标识已完成对压缩目标的设置
MemoryStream mStream = new MemoryStream();//内存流
byte[] comCache = new byte[byteCache.Length];//压缩缓存
while (!deFlater.IsFinished)
{
int comLength = deFlater.Deflate(comCache);//压缩(通过SetInput)指定字节数组,并将压缩结果写入缓存
mStream.Write(comCache, 0, comLength);//把压缩缓存写入内存流
}
byte[] comResult = mStream.ToArray();//从内存流中把压缩结果转换回字节数组
mStream.Close();//关闭内存流
return comResult;//返回压缩结果
}
//public static byte[] DeCompressByte(byte[] byteCache)
//{
// Inflater inflater = new Inflater();//创建解压实例
// inflater.SetInput(byteCache);//设置需要解压的字节
// MemoryStream mStream = new MemoryStream();//创建内存流实例
// byte[] inflaterCache = new byte[9999999999];//创建解压缓存字节数据组
// while (!inflater.IsFinished)//开始解压
// {
// int inflaterLength = inflater.Inflate(inflaterCache);//解压写入缓存数组
// mStream.Write(inflaterCache, 0, inflaterLength);//解压后的缓存数据写入内存流
// }
// byte[] deComResult = mStream.ToArray();//解压结果
// mStream.Close();//关闭内容流服务
// return deComResult;//返回解压的后的字节
//}
}
}
读:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using ICSharpCode.SharpZipLib.Zip.Compression;
using System.IO;
namespace ReadFileTest
{
class Program
{
public struct Tick
{
public string s_Date;
public int i_length;
public int i_Offset;
}
public static Dictionary<string, Tick> DataDic = new Dictionary<string, Tick>();
static void Main(string[] args)
{
GetFileInfo("201408151846", "CU");
Console.ReadLine();
}
public static void GetFileInfo(string s_Time, string s_Type)
{
string FileDataName = s_Type + "_" + s_Time.Substring(0, 8) + "." + "data";
string FileIndexName = s_Type + "_" + s_Time.Substring(0, 8) + "." + "index";
Console.WriteLine(FileDataName);
Console.WriteLine(FileIndexName);
int i = 0;
int i_Offset = 0;
using (StreamReader sr = new StreamReader(@"E:\Documents\Visual Studio 2010\Projects\WriteFileTest\WriteFileTest\bin\Debug\data\" + FileIndexName))
{
String line;
while ((line = sr.ReadLine()) != null)
{
Tick getTick = new Tick();
string s1 = line.Substring(0, line.IndexOf("|"));
string s2 = line.Substring(line.IndexOf("|") + 1);
Console.WriteLine("第" + i + "条的时间是:" + s1 + " 长度是:" + s2 + "\t\t偏移:" + i_Offset.ToString());
getTick.i_length = Int32.Parse(s2);
getTick.s_Date = s1;
getTick.i_Offset = i_Offset;
i_Offset = i_Offset + Int32.Parse(s2);
DataDic.Add("CU" + s1, getTick);
getData(FileDataName, getTick.i_length, getTick.i_Offset);
i++;
}
Console.WriteLine(i.ToString());
}
return;
}
public static string getData(string FileName, int i_length, long l_Offset)
{
string s_FileName = @"E:\Documents\Visual Studio 2010\Projects\WriteFileTest\WriteFileTest\bin\Debug\data\" + FileName;
FileStream fs = new FileStream(s_FileName, FileMode.Open, FileAccess.Read);
byte[] data= new byte[i_length];
fs.Seek(l_Offset, SeekOrigin.Begin);
fs.Read(data, 0, data.Length);
string s=Encoding.UTF8.GetString(DeCompressByte(data));
fs.Close();
return s;
}
public static byte[] DeCompressByte(byte[] byteCache)
{
//return byteCache;
Inflater inflater = new Inflater();//创建解压实例
inflater.SetInput(byteCache);//设置需要解压的字节
MemoryStream mStream = new MemoryStream();//创建内存流实例
byte[] inflaterCache = new byte[99999];//创建解压缓存字节数据组
while (!inflater.IsFinished)//开始解压
{
int inflaterLength = inflater.Inflate(inflaterCache);//解压写入缓存数组
mStream.Write(inflaterCache, 0, inflaterLength);//解压后的缓存数据写入内存流
}
byte[] deComResult = mStream.ToArray();//解压结果
mStream.Close();//关闭内容流服务
return deComResult;//返回解压的后的字节
}
}
}