参考varint编码:
http://www.360doc.com/content/14/0325/16/15064667_363619247.shtml
http://blog.csdn.net/flyfish1986/article/details/49020131
为了减少int等类型内存占用,采用了精巧的压缩算法。
Varint 中的每个 byte 的最高位 bit 是标识位有特殊的含义。 如果该位为 1,表示后续的 byte 也是该数字的一部分,
如果该位为 0,则结束。 其他的 7 个 bit 都用来表示数字。
举例说明 Varint编码和解码过程
1000 为例
编码过程
1000 二进制为 1111101000
取右边7bit,1101000
剩下的是111
小端存储要转换位置
字节1:01101000
字节2: 00000111
每个字节位为标识1,还有后续。所以字节1最高是1,字节2最高位为0
编码结果是: 11101000 00000111
解码过程
11101000 00000111
高位1有后续,与下一个字节一同表示一个数值
1101000 00000111
字节交换位置结果是: 111 1101000
// Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // Endian-neutral encoding: // * Fixed-length numbers are encoded with least-significant byte first // * In addition we support variable length "varint" encoding // * Strings are encoded prefixed by their length in varint format #ifndef STORAGE_LEVELDB_UTIL_CODING_H_ #define STORAGE_LEVELDB_UTIL_CODING_H_ #include <stdint.h> #include <string.h> #include <string> #include "leveldb/slice.h" #include "port/port.h" namespace leveldb { // Standard Put... routines append to a string extern void PutFixed32(std::string* dst, uint32_t value); extern void PutFixed64(std::string* dst, uint64_t value); extern void PutVarint32(std::string* dst, uint32_t value); extern void PutVarint64(std::string* dst, uint64_t value); extern void PutLengthPrefixedSlice(std::string* dst, const Slice& value); // Standard Get... routines parse a value from the beginning of a Slice // and advance the slice past the parsed value. extern bool GetVarint32(Slice* input, uint32_t* value); extern bool GetVarint64(Slice* input, uint64_t* value); extern bool GetLengthPrefixedSlice(Slice* input, Slice* result); // Pointer-based variants of GetVarint... These either store a value // in *v and return a pointer just past the parsed value, or return // NULL on error. These routines only look at bytes in the range // [p..limit-1] extern const char* GetVarint32Ptr(const char* p,const char* limit, uint32_t* v); extern const char* GetVarint64Ptr(const char* p,const char* limit, uint64_t* v); // Returns the length of the varint32 or varint64 encoding of "v" extern int VarintLength(uint64_t v); // Lower-level versions of Put... that write directly into a character buffer // REQUIRES: dst has enough space for the value being written extern void EncodeFixed32(char* dst, uint32_t value); extern void EncodeFixed64(char* dst, uint64_t value); // Lower-level versions of Put... that write directly into a character buffer // and return a pointer just past the last byte written. // REQUIRES: dst has enough space for the value being written extern char* EncodeVarint32(char* dst, uint32_t value); extern char* EncodeVarint64(char* dst, uint64_t value); // Lower-level versions of Get... that read directly from a character buffer // without any bounds checking. inline uint32_t DecodeFixed32(const char* ptr) { if (port::kLittleEndian) { // Load the raw bytes uint32_t result; memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load return result; } else { return ((static_cast<uint32_t>(static_cast<unsigned char>(ptr[0]))) | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[1])) << 8) | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[2])) << 16) | (static_cast<uint32_t>(static_cast<unsigned char>(ptr[3])) << 24)); } } inline uint64_t DecodeFixed64(const char* ptr) { if (port::kLittleEndian) { // Load the raw bytes uint64_t result; memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load return result; } else { uint64_t lo = DecodeFixed32(ptr); uint64_t hi = DecodeFixed32(ptr + 4); return (hi << 32) | lo; } } // Internal routine for use by fallback path of GetVarint32Ptr extern const char* GetVarint32PtrFallback(const char* p, const char* limit, uint32_t* value); inline const char* GetVarint32Ptr(const char* p, const char* limit, uint32_t* value) { if ( p < limit ) { uint32_t result = *(reinterpret_cast<const unsigned char*>(p)); if ( ( result & 128 ) == 0 ) { *value = result; return p + 1; } } return GetVarint32PtrFallback( p, limit, value ); } } // namespace leveldb #endif // STORAGE_LEVELDB_UTIL_CODING_H_
// Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "util/coding.h" namespace leveldb { // 固定长度编码 void EncodeFixed32( char* buf, uint32_t value ) { if ( port::kLittleEndian ) { memcpy( buf, &value, sizeof ( value ) ); } else { buf[0] = value & 0xff; buf[1] = (value >> 8) & 0xff; buf[2] = (value >> 16) & 0xff; buf[3] = (value >> 24) & 0xff; } } void EncodeFixed64( char* buf, uint64_t value ) { if ( port::kLittleEndian ) { memcpy( buf, &value, sizeof ( value ) ); } else { buf[0] = value & 0xff; buf[1] = (value >> 8) & 0xff; buf[2] = (value >> 16) & 0xff; buf[3] = (value >> 24) & 0xff; buf[4] = (value >> 32) & 0xff; buf[5] = (value >> 40) & 0xff; buf[6] = (value >> 48) & 0xff; buf[7] = (value >> 56) & 0xff; } } void PutFixed32(std::string* dst, uint32_t value) { char buf[sizeof(value)]; EncodeFixed32(buf, value); dst->append(buf, sizeof(buf)); } void PutFixed64(std::string* dst, uint64_t value) { char buf[sizeof(value)]; EncodeFixed64(buf, value); dst->append(buf, sizeof(buf)); } // 变长编码 char* EncodeVarint32(char* dst, uint32_t v) { // Operate on characters as unsigneds unsigned char* ptr = reinterpret_cast<unsigned char*>(dst); static const int B = 128; if (v < (1<<7)) { *(ptr++) = v; } else if (v < (1<<14)) { *(ptr++) = v | B; *(ptr++) = v>>7; } else if (v < (1<<21)) { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = v>>14; } else if (v < (1<<28)) { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = (v>>14) | B; *(ptr++) = v>>21; } else { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = (v>>14) | B; *(ptr++) = (v>>21) | B; *(ptr++) = v>>28; } return reinterpret_cast<char*>(ptr); } void PutVarint32( std::string* dst, uint32_t v ) { char buf[5]; char* ptr = EncodeVarint32( buf, v ); dst->append( buf, ptr - buf ); } char* EncodeVarint64(char* dst, uint64_t v) { static const int B = 128; unsigned char* ptr = reinterpret_cast<unsigned char*>(dst); while ( v >= B ) { *(ptr++) = (v & (B-1)) | B; v >>= 7; } *(ptr++) = static_cast<unsigned char>(v); return reinterpret_cast<char*>(ptr); } void PutVarint64(std::string* dst, uint64_t v) { char buf[10]; char* ptr = EncodeVarint64(buf, v); dst->append( buf, ptr - buf ); } void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { PutVarint32(dst, value.size()); dst->append(value.data(), value.size()); } int VarintLength( uint64_t v ) { int len = 1; while ( v >= 128 ) { v >>= 7; len++; } return len; } const char* GetVarint32PtrFallback(const char* p, const char* limit, uint32_t* value) { uint32_t result = 0; for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { uint32_t byte = *(reinterpret_cast<const unsigned char*>(p)); p++; if (byte & 128) { // More bytes are present result |= ((byte & 127) << shift); } else { result |= (byte << shift); *value = result; return reinterpret_cast<const char*>(p); } } return NULL; } bool GetVarint32(Slice* input, uint32_t* value) { const char* p = input->data(); const char* limit = p + input->size(); const char* q = GetVarint32Ptr( p, limit, value ); if ( q == NULL ) { return false; } else { *input = Slice( q, limit - q ); return true; } } const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { uint64_t result = 0; for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { uint64_t byte = *(reinterpret_cast<const unsigned char*>(p)); p++; if (byte & 128) { // More bytes are present result |= ((byte & 127) << shift); } else { result |= (byte << shift); *value = result; return reinterpret_cast<const char*>(p); } } return NULL; } bool GetVarint64(Slice* input, uint64_t* value) { const char* p = input->data(); const char* limit = p + input->size(); const char* q = GetVarint64Ptr(p, limit, value); if (q == NULL) { return false; } else { *input = Slice(q, limit - q); return true; } } const char* GetLengthPrefixedSlice(const char* p, const char* limit, Slice* result) { uint32_t len; p = GetVarint32Ptr(p, limit, &len); if (p == NULL) return NULL; if (p + len > limit) return NULL; *result = Slice(p, len); return p + len; } bool GetLengthPrefixedSlice(Slice* input, Slice* result) { uint32_t len; if (GetVarint32(input, &len) && input->size() >= len) { *result = Slice(input->data(), len); input->remove_prefix(len); return true; } else { return false; } } } // namespace leveldb