最近在优化实时数据库(参考influxdb写的)存储性能时,遇到了要将bool, int, long, double, string 序列化为字节数组(mashal),然后再Snappy压缩的部分,通过visual studio 2017中的性能探查器,发现mashal CPU占比2.23%,有优化潜力,于是做了如下尝试:
原方案: List<byte> + BitConverter.GetBytes()
集中改进的思路:
- 采用ArrayPool<T>避免每次分配数组的内存;
- 采用BinaryWriter+MemoryStream ;
- 用BitConverter.TryWriteBytes + Span<T>;
- 自己实现一个ByteWriter。
经过2个多小时的尝试,发现2、3比原方案CPU占比反而更高了,查了一下bing,貌似BitConverter.GetBytes()在底层用指针实现的,效率颇高,也许.net core2.1 里面的BinaryWriter还没优化吧。
然后看到一篇读写二进制最快的方法的stackoverflow,核心思想是用位移来进行数据的类型转换,受到启发,决定自己实现一个ByteWriter,实测CPU占为1.12%,即下降50%,注意:ByteWriter.EndWrite返回的byte[]在使用(如Stream.Write())后,应调用Release。当然,通过unsafe指针操作性能还有提升空间。
using System.Buffers;
namespace Arim.Plat.Trend.Engine
{
public class ByteWriter
{
byte[] buffer;
int i;
public ByteWriter(int length)
{
Length = length;
buffer = ArrayPool<byte>.Shared.Rent(length);
i = 0;
}
public int Length { get; private set; }
public void Write(byte v)
{
buffer[i++] = v;
}
public void Write(bool v)
{
if (v) buffer[i++] = 0x01;
else buffer[i++] = 0x00;
}
public void Write(ushort v)
{
buffer[i++] = (byte)v;
buffer[i++] = (byte)(v >> 8);
}
public void Write(int v)
{
buffer[i++] = (byte)v;
buffer[i++] = (byte)(v >> 8);
buffer[i++] = (byte)(v >> 16);
buffer[i++] = (byte)(v >> 24);
}
public void Write(uint v)
{
buffer[i++] = (byte)v;
buffer[i++] = (byte)(v >> 8);
buffer[i++] = (byte)(v >> 16);
buffer[i++] = (byte)(v >> 24);
}
public void Write(long v)
{
buffer[i++] = (byte)v;
buffer[i++] = (byte)(v >> 8);
buffer[i++] = (byte)(v >> 16);
buffer[i++] = (byte)(v >> 24);
buffer[i++] = (byte)(v >> 32);
buffer[i++] = (byte)(v >> 40);
buffer[i++] = (byte)(v >> 48);
buffer[i++] = (byte)(v >> 56);
}
public void Write(ulong v)
{
buffer[i++] = (byte)v;
buffer[i++] = (byte)(v >> 8);
buffer[i++] = (byte)(v >> 16);
buffer[i++] = (byte)(v >> 24);
buffer[i++] = (byte)(v >> 32);
buffer[i++] = (byte)(v >> 40);
buffer[i++] = (byte)(v >> 48);
buffer[i++] = (byte)(v >> 56);
}
public void Write(double v)
{
Write(FloatEncoder.Float64bits(v));
}
public void Write(string v)
{
byte[] strBytes = System.Text.Encoding.Default.GetBytes(v);
int len = strBytes.Length;
Write(len);
for(int j=0;j<len;j++)
{
buffer[i++] = strBytes[j];
}
}
public void Write(byte[] v)
{
for (int j = 0; j < v.Length; j++)
{
buffer[i++] = v[j];
}
}
public byte[] EndWrite()
{
return buffer;
}
public void Release()
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
}