序列化有很多种方式,最简单的就是gob,但是最慢,如果我们使用bigCache之类的库,则需要自己做序列化以及反序列化,
高并发环境下还是很重要的,各种对比参考:
我这里要使用就是最直接的序列化,按照小端直接写到字节流,网上很多的例子都是这样的:
type Counter struct {
Count uint64
LastCount uint64
}
func Counter2Buf(c Counter) []byte {
buf := bytes.NewBuffer([]byte{})
buf.Grow(16)
binary.Write(buf, binary.LittleEndian, c.Count)
binary.Write(buf, binary.LittleEndian, c.LastCount)
return buf.Bytes()
}
func Buf2Counter(b []byte) (*Counter, error) {
if len(b) < 16 {
return nil, fmt.Errorf("len is too short")
}
c := Counter{0, 0}
bytesBuffer := bytes.NewBuffer(b)
binary.Read(bytesBuffer, binary.LittleEndian, &c.Count)
binary.Read(bytesBuffer, binary.LittleEndian, &c.LastCount)
return &c, nil
}
而实际上性能依然很差,比protobuf还要差,比如我需要做100万次操作,结果如下:
编码:106 ms
解码:86 ms
对应的1单次操作平均:116ms和86ms,protobuf压缩也不过如此,
其实这里还可以更快,直接操作[]byte,
// 这里按照系统库的方式抄一下,直接添加一个偏移就可以了
func putUint64Little(b []byte, v uint64, off int) {
b[0+off] = byte(v)
b[1+off] = byte(v >> 8)
b[2+off] = byte(v >> 16)
b[3+off] = byte(v >> 24)
b[4+off] = byte(v >> 32)
b[5+off] = byte(v >> 40)
b[6+off] = byte(v >> 48)
b[7+off] = byte(v >> 56)
}
func Couter2Bytes(c Counter) []byte {
buff := make([]byte, 16)
binary.LittleEndian.PutUint64(buff, c.Count)
putUint64Little(buff, c.LastCount, 8)
return buff
}
///
// 这里按照系统库的方式抄一下,直接添加一个偏移就可以了
func Uint64Litter(b []byte, off int) uint64 {
_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[0+off]) | uint64(b[1+off])<<8 | uint64(b[2+off])<<16 | uint64(b[3+off])<<24 |
uint64(b[4+off])<<32 | uint64(b[5+off])<<40 | uint64(b[6+off])<<48 | uint64(b[7+off])<<56
}
// 字节转换成整形
func BytesToCounter(b []byte) (*Counter, error) {
if len(b) < 16 {
return nil, fmt.Errorf("len is too short")
}
c := Counter{binary.LittleEndian.Uint64(b),
Uint64Litter(b, 8)}
return &c, nil
}
这时候,我们执行测试100万次编码测试,
编码:19 ms
解码: 18ms
平均单次耗时:编码19ns,解码18ns
整体性能提高了5倍;
后记:有人做过相关对比测试:go语言序列化json/gob/msgp/protobuf性能对比 - 知乎
对比帖子中的性能测试,我们直接做一个写测试:
type Doc struct {
Doc_id uint32 // 4B
School_level uint32
Active uint32
Work_age uint32
Vip bool // 1B
Chat bool
Position string // 4B + bytes
Company string
City string
}
func Doc2Bytes(doc *Doc) []byte {
length := 16 + 2 + 12 + len(doc.Position) + len(doc.Company) + len(doc.City)
buf := make([]byte, length)
putUint32Little(buf, 0, doc.Doc_id)
putUint32Little(buf, 4, doc.School_level)
putUint32Little(buf, 8, doc.Active)
putUint32Little(buf, 12, doc.Work_age)
putBool(buf, 16, doc.Vip)
putBool(buf, 17, doc.Chat)
off := putString(buf, 18, doc.Position)
off = putString(buf, off, doc.Company)
putString(buf, off, doc.City)
return buf
}
func DocFromByte(b []byte) *Doc {
doc := Doc{
Doc_id: 0,
School_level: 0,
Active: 0,
Work_age: 0,
Vip: false,
Chat: false,
Position: "",
Company: "",
City: "",
}
doc.Doc_id = Uint32Little(b, 0)
doc.School_level = Uint32Little(b, 4)
doc.Active = Uint32Little(b, 8)
doc.Work_age = Uint32Little(b, 12)
doc.Vip = BoolLitter(b, 16)
doc.Chat = BoolLitter(b, 17)
var off int = 0
doc.Position, off = StringLittle(b, 18)
doc.Company, off = StringLittle(b, off)
doc.City, off = StringLittle(b, off)
return &doc
}
func putUint32Little(b []byte, off int, v uint32) {
// 这样比指针更快些,
b[0+off] = byte(v)
b[1+off] = byte(v >> 8)
b[2+off] = byte(v >> 16)
b[3+off] = byte(v >> 24)
//pInt := unsafe.Pointer(&v)
//bInt := (*[4]byte)(pInt)
//copy(b[off:], (*bInt)[0:])
}
func putBool(b []byte, off int, v bool) {
if v {
b[0+off] = byte(1)
} else {
b[0+off] = byte(0)
}
}
func str2bytes(s string) []byte {
x := (*[2]uintptr)(unsafe.Pointer(&s))
h := [3]uintptr{x[0], x[1], x[1]}
return *(*[]byte)(unsafe.Pointer(&h))
}
func bytes2str(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}
func putString(b []byte, off int, v string) int {
length := uint32(len(v))
putUint32Little(b, off, length)
off = off + 4
// TODO: 优化这里
//buf := []byte(v)
//for i, v := range buf {
// b[off+4+i] = v
//}
//return off + 4 + len(v)
// test it
x := (*[2]uintptr)(unsafe.Pointer(&v))
h := [3]uintptr{x[0], x[1], x[1]}
bStr := *(*[]byte)(unsafe.Pointer(&h))
//bStr := str2bytes(v)
copy(b[off:], bStr[0:])
return off + len(v)
}
func Uint32Little(b []byte, off int) uint32 {
return uint32(b[0+off]) | uint32(b[1+off])<<8 | uint32(b[2+off])<<16 | uint32(b[3+off])<<24
}
func BoolLitter(b []byte, off int) bool {
a := int8(b[0+off])
if a != 0 {
return true
}
return false
}
func StringLittle(b []byte, off int) (string, int) {
length := Uint32Little(b, off)
bPart := b[off+4 : off+4+int(length)]
str := bytes2str(bPart)
return str, off + 4 + int(length)
}
func testSeri() {
doc := Doc{1, 2, 3, 4,
false, true,
"abcdefg",
"XXXX",
"zz",
}
b := Doc2Bytes(&doc)
fmt.Println(b)
doc1 := DocFromByte(b)
fmt.Println(doc1)
}
测试结果是平均42ns,比protobuf还要快一点点;