以hadoop的avro实现原理做引子：
varo的测试样例程序
#include <complex>
#include "avro/Encoder.hh"
#include "avro/Decoder.hh"
#include "avro/Specific.hh"
namespace avro {
template<typename T>
struct codec_traits<std::complex<T> > {
static void encode(Encoder& e, const std::complex<T>& c) {
avro::encode(e, std::real(c));
avro::encode(e, std::imag(c));
}
static void decode(Decoder& d, std::complex<T>& c) {
T re, im;
avro::decode(d, re);
avro::decode(d, im);
c = std::complex<T>(re, im);
}
};
}
int
main()
{
std::auto_ptr<avro::OutputStream> out = avro::memoryOutputStream();
avro::EncoderPtr e = avro::binaryEncoder();
e->init(*out);
std::complex<double> c1(1.0, 2.0);
avro::encode(*e, c1);
std::auto_ptr<avro::InputStream> in = avro::memoryInputStream(*out);
avro::DecoderPtr d = avro::binaryDecoder();
d->init(*in);
std::complex<double> c2;
avro::decode(*d, c2);
std::cout << '(' << std::real(c2) << ", " << std::imag(c2) << ')' << std::endl;
return 0;
}这个样例相对简单。
其中template <typename T>
struct codec_traits { };
为avro对不同数据类型存储实现的模板类特化。
由于avro的基础库类未有提供std::complex<double>的特化说明，故在样例函数添加avro作用域的complex类型说明，并将复数类型的特化函数转化为avro提供的基础数据类型的特化
特化函数定义在avro api/specific.hh 头文件中定义。样例这里使用了 143行定义的double类型处理
template <> struct codec_traits<double> {
/**
* Encodes a given value.
*/
static void encode(Encoder& e, double d) {
e.encodeDouble(d);
}
/**
* Decodes into a given value.
*/
static void decode(Decoder& d, double& dbl) {
dbl = d.decodeDouble();
}
};
这里说明下avro内部存储数据的结构：
由于样例中使用的是 二进制编制的内存存储形式（相对于文件存储）
std::auto_ptr<avro::OutputStream> out = avro::memoryOutputStream();
avro::EncoderPtr e = avro::binaryEncoder();
其中 ：memoryOutputStream 数据保存内存存储类
binaryEncoder 存储数据写入管理类 --- 对应Impl/binaryEncoder.cc文件
double写入内存函数实现：
void BinaryEncoder::encodeDouble(double d)
{
const uint8_t* p = reinterpret_cast<const uint8_t*>(&d);
out_.writeBytes(p, sizeof(double));
}
out为MemoryOutputStream管理类。对应文件实现impl/stream.cc 文件
其存储形式如下图。如上例中c1(1.0,2.0)。其中图示每一个方格为一个double存储单元。
写入逻辑实现为：
1.检测当前空间，如上图中一行是否足够double写入（注每次申请行为固定个数double存储单元）。否申请新的空间（即如上 图一行）
2.写入double数据，并记录当前写入地址及最大可写入地址。
以上为数据写入的实现，反之取出，只需要按顺序读取数值出来即可。
难点：avro特化函数对指定数据类型的处理
另外提供两种序列化和反序列化实现：
第一种：来自哥们写的。特复杂也很难理解，（我也看了好几天才懂~~），仅给出部分关键文件源码。
想看完整源码 可在github下载 ：https://github.com/xyz347/x2struct
以json序列化为例源码示例：
X2Struct.hh 这个文件是关键，以宏遍历的形式对结构体成员进行逐个成员展开处理。如果想实现非json存储数据类型的数据序列化和反序列化也以这个文件为基础构建。
特别注意：结构体成员展开为二层次展开，（怎么理解，看代码~~~~）。结构体成员最大展开个数为 32*32
#ifndef __X_TO_STRUCT_HPP
#define __X_TO_STRUCT_HPP
#include <string>
#include <set>
#include <stdexcept>
#include "xobj.hpp"
#include "xmlstr.hpp"
#include "jsoncfgstr.hpp"
#ifdef XTOSTRUCT_GEN_GOLANG_CODE
#include <typeinfo>
#include "gostr.hpp"
#endif
//#include <iostream>
//using namespace std;
namespace x2struct {
#define X2STRUCT_OPT_ME "me" // must exist
class X {
public:
template <typename TYPE>
static bool loadjson(const std::string&str, TYPE&t, bool isfile=true) {
JsonObj obj(str, isfile);
t.__x_to_struct(obj);
return true;
}
template <typename TYPE>
static std::string tojson(const TYPE&t, const std::string&root, bool newline=true, int space=4) {
JsonCfgStr obj(true, newline, space);
t.__struct_to_str(obj, root, 0);
return obj.toStr();
}
};
#define X_STRUCT_FUNC_TOX_BEGIN \
private: \
std::set<std::string> __x_has_string; \
public: \
bool xhas(const std::string& name) const { \
return __x_has_string.find(name)!=__x_has_string.end(); \
} \
bool __x_condition(x2struct::XObj& obj, const std::string&name) { \
(void)obj;(void)name; \
return true; \
} \
void __x_to_struct(x2struct::XObj& obj) {
#define X_STRUCT_ACT_TOX_O(M) \
if (obj.has(#M)) { \
obj[#M].convert(M); \
__x_has_string.insert(#M); \
}
#define X_STRUCT_ACT_TOX_M(M) \
if (obj.has(#M)) { \
obj[#M].convert(M); \
__x_has_string.insert(#M); \
} else { \
obj.me_exception(#M); \
}
// aliase name
#define X_STRUCT_ACT_TOX_A(M, A_NAME) \
{ \
bool me = false; \
std::string __alias__name__ = obj.hasa(#M, A_NAME, me); \
if (!__alias__name__.empty()) { \
obj[__alias__name__].convert(M); \
__x_has_string.insert(#M); \
} else if (me) { \
obj.me_exception(#M); \
} \
}
#define X_STRUCT_FUNC_TOX_END }
// struct to string
#define X_STRUCT_FUNC_TOS_BEGIN \
template <class CLASS> \
void __struct_to_str(CLASS& obj, const std::string&root, int splen) const { \
int index = 0; \
obj.begin(root, splen);
#define X_STRUCT_ACT_TOS_O(M) \
obj.convert(#M, M, splen+obj.space(), index++);
#define X_STRUCT_ACT_TOS_A(M, A_NAME) \
obj.convert(obj.fieldName(#M,A_NAME), M, splen+obj.space(), index++);
#define X_STRUCT_FUNC_TOS_END \
obj.end(root,splen); \
}
// struct to golang code
#define X_STRUCT_FUNC_TOG_BEGIN \
std::string __struct_to_go(x2struct::GoCode &obj) const { \
obj.set_type_id_name(typeid(*this).name());
#define X_STRUCT_ACT_TOG_O(M) \
obj.convert(M, #M);
#define X_STRUCT_ACT_TOG_A(M, A_NAME) \
obj.convert(M, #M, A_NAME);
#define X_STRUCT_FUNC_TOG_END \
return obj.str(); \
}
// lenght should equal X_STRUCT_N
#define X_DEC_LIST _32,_31,_30,_29,_28,_27,_26,_25,_24,_23,_22,_21,_20,_19,_18,_17,_16,_15,_14,_13,_12,_11,_10,_9,_8,_7,_6,_5,_4,_3,_2,_1
// expand macro depend on parameters number
#define X_STRUCT_N(ACTION,_1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11,_12,_13,_14,_15,_16,_17,_18,_19,_20,_21,_22,_23,_24,_25,_26,_27,_28,_29,_30,_31,_32,N,...) ACTION##N
/*
work with X_STRUCT_N to expand macro
struct MyX {
int a;
std::string b;
double c;
XTOSTRUCT(A(a,"_id"), O(b,c));
};
there'r two layer in this macro:
l1: A(a,"_id"), O(b,c)
l2: a,"_id"; b,c
so we need expand twice
*/
// wrap X_STRUCT_N. so we can use X_DEC_LIST
#define X_STRUCT_WRAP_L1(ACT, LIST, ...) X_STRUCT_N(X_STRUCT_L1, __VA_ARGS__, LIST)(ACT, __VA_ARGS__)
#define X_STRUCT_WRAP_L2(ACT, LIST, ...) X_STRUCT_N(X_STRUCT_L2, __VA_ARGS__, LIST)(ACT, __VA_ARGS__)
#define X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_##ACT##M
#define X_STRUCT_L1_1(ACT, M) X_STRUCT_L1_DEF(ACT, M)
#define X_STRUCT_L1_2(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_1(ACT, __VA_ARGS__)
#define X_STRUCT_L1_3(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_2(ACT, __VA_ARGS__)
#define X_STRUCT_L1_4(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_3(ACT, __VA_ARGS__)
#define X_STRUCT_L1_5(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_4(ACT, __VA_ARGS__)
#define X_STRUCT_L1_6(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_5(ACT, __VA_ARGS__)
#define X_STRUCT_L1_7(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_6(ACT, __VA_ARGS__)
#define X_STRUCT_L1_8(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_7(ACT, __VA_ARGS__)
#define X_STRUCT_L1_9(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_8(ACT, __VA_ARGS__)
#define X_STRUCT_L1_10(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_9(ACT, __VA_ARGS__)
#define X_STRUCT_L1_11(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_10(ACT, __VA_ARGS__)
#define X_STRUCT_L1_12(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_11(ACT, __VA_ARGS__)
#define X_STRUCT_L1_13(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_12(ACT, __VA_ARGS__)
#define X_STRUCT_L1_14(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_13(ACT, __VA_ARGS__)
#define X_STRUCT_L1_15(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_14(ACT, __VA_ARGS__)
#define X_STRUCT_L1_16(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_15(ACT, __VA_ARGS__)
#define X_STRUCT_L1_17(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_16(ACT, __VA_ARGS__)
#define X_STRUCT_L1_18(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_17(ACT, __VA_ARGS__)
#define X_STRUCT_L1_19(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_18(ACT, __VA_ARGS__)
#define X_STRUCT_L1_20(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_19(ACT, __VA_ARGS__)
#define X_STRUCT_L1_21(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_20(ACT, __VA_ARGS__)
#define X_STRUCT_L1_22(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_21(ACT, __VA_ARGS__)
#define X_STRUCT_L1_23(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_22(ACT, __VA_ARGS__)
#define X_STRUCT_L1_24(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_23(ACT, __VA_ARGS__)
#define X_STRUCT_L1_25(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_24(ACT, __VA_ARGS__)
#define X_STRUCT_L1_26(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_25(ACT, __VA_ARGS__)
#define X_STRUCT_L1_27(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_26(ACT, __VA_ARGS__)
#define X_STRUCT_L1_28(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_27(ACT, __VA_ARGS__)
#define X_STRUCT_L1_29(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_28(ACT, __VA_ARGS__)
#define X_STRUCT_L1_30(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_29(ACT, __VA_ARGS__)
#define X_STRUCT_L1_31(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_30(ACT, __VA_ARGS__)
#define X_STRUCT_L1_32(ACT, M,...) X_STRUCT_L1_DEF(ACT, M) X_STRUCT_L1_31(ACT, __VA_ARGS__)
#define X_STRUCT_L2_DEF(ACT, M) X_STRUCT_ACT_##ACT(M)
#define X_STRUCT_L2_1(ACT, M) X_STRUCT_L2_DEF(ACT, M)
#define X_STRUCT_L2_2(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_1(ACT, __VA_ARGS__)
#define X_STRUCT_L2_3(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_2(ACT, __VA_ARGS__)
#define X_STRUCT_L2_4(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_3(ACT, __VA_ARGS__)
#define X_STRUCT_L2_5(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_4(ACT, __VA_ARGS__)
#define X_STRUCT_L2_6(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_5(ACT, __VA_ARGS__)
#define X_STRUCT_L2_7(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_6(ACT, __VA_ARGS__)
#define X_STRUCT_L2_8(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_7(ACT, __VA_ARGS__)
#define X_STRUCT_L2_9(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_8(ACT, __VA_ARGS__)
#define X_STRUCT_L2_10(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_9(ACT, __VA_ARGS__)
#define X_STRUCT_L2_11(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_10(ACT, __VA_ARGS__)
#define X_STRUCT_L2_12(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_11(ACT, __VA_ARGS__)
#define X_STRUCT_L2_13(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_12(ACT, __VA_ARGS__)
#define X_STRUCT_L2_14(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_13(ACT, __VA_ARGS__)
#define X_STRUCT_L2_15(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_14(ACT, __VA_ARGS__)
#define X_STRUCT_L2_16(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_15(ACT, __VA_ARGS__)
#define X_STRUCT_L2_17(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_16(ACT, __VA_ARGS__)
#define X_STRUCT_L2_18(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_17(ACT, __VA_ARGS__)
#define X_STRUCT_L2_19(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_18(ACT, __VA_ARGS__)
#define X_STRUCT_L2_20(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_19(ACT, __VA_ARGS__)
#define X_STRUCT_L2_21(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_20(ACT, __VA_ARGS__)
#define X_STRUCT_L2_22(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_21(ACT, __VA_ARGS__)
#define X_STRUCT_L2_23(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_22(ACT, __VA_ARGS__)
#define X_STRUCT_L2_24(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_23(ACT, __VA_ARGS__)
#define X_STRUCT_L2_25(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_24(ACT, __VA_ARGS__)
#define X_STRUCT_L2_26(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_25(ACT, __VA_ARGS__)
#define X_STRUCT_L2_27(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_26(ACT, __VA_ARGS__)
#define X_STRUCT_L2_28(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_27(ACT, __VA_ARGS__)
#define X_STRUCT_L2_29(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_28(ACT, __VA_ARGS__)
#define X_STRUCT_L2_30(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_29(ACT, __VA_ARGS__)
#define X_STRUCT_L2_31(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_30(ACT, __VA_ARGS__)
#define X_STRUCT_L2_32(ACT, M,...) X_STRUCT_L2_DEF(ACT, M) X_STRUCT_L2_31(ACT, __VA_ARGS__)
// L1 action define. O M A
// string to struct
#define X_STRUCT_L1_TOX_O(...) X_STRUCT_WRAP_L2(TOX_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOX_M(...) X_STRUCT_WRAP_L2(TOX_M, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOX_A(M,A) X_STRUCT_ACT_TOX_A(M,A)
// struct to string
#define X_STRUCT_L1_TOS_O(...) X_STRUCT_WRAP_L2(TOS_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOS_M X_STRUCT_L1_TOS_O
#define X_STRUCT_L1_TOS_A(M,A) X_STRUCT_ACT_TOS_A(M,A)
// struct to golang code
#define X_STRUCT_L1_TOG_O(...) X_STRUCT_WRAP_L2(TOG_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOG_M X_STRUCT_L1_TOG_O
#define X_STRUCT_L1_TOG_A(M,A) X_STRUCT_ACT_TOG_A(M,A)
#ifdef XTOSTRUCT_GEN_GOLANG_CODE
#define XTOSTRUCT(...) \
X_STRUCT_FUNC_TOX_BEGIN X_STRUCT_WRAP_L1(TOX_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOX_END \
X_STRUCT_FUNC_TOS_BEGIN X_STRUCT_WRAP_L1(TOS_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOS_END \
X_STRUCT_FUNC_TOG_BEGIN X_STRUCT_WRAP_L1(TOG_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOG_END
#else
#define XTOSTRUCT(...) \
X_STRUCT_FUNC_TOX_BEGIN X_STRUCT_WRAP_L1(TOX_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOX_END \
X_STRUCT_FUNC_TOS_BEGIN X_STRUCT_WRAP_L1(TOS_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOS_END
#endif
#define XTOSTRUCT_CONDITION() bool __x_condition(x2struct::XObj& obj, const char* name)
#define XTOSTRUCT_CONDITION_EQ(attr1, attr2) \
bool __x_condition(x2struct::XObj& obj, const char* name) { \
return obj.attribute(attr1)==obj.attribute(attr2); \
}
}
#endif
jsonstrcfg.h json序列化和反序列化接口定义
* Copyright (C) 2017 YY Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef __X_TO_STRUCT_JSON_CFG_STR_HPP
#define __X_TO_STRUCT_JSON_CFG_STR_HPP
#include <stdint.h>
#include <string>
#include <map>
#include <vector>
#include <set>
#include <sstream>
#include <boost/lexical_cast.hpp>
#include "xstr.hpp"
namespace x2struct {
class JsonCfgStr:public XStr {
public:
JsonCfgStr(bool isjson, bool newline=false, int splen=4);
~JsonCfgStr();
public:
void begin(const std::string&root, int splen);
void end(const std::string&root, int splen);
int space();
std::string toStr() const;
public:
virtual void convert(const std::string&name, const std::string& data, int space, int index);
// base type
void convert(const std::string&name, int16_t data, int splen, int index);
void convert(const std::string&name, uint16_t data, int splen, int index);
void convert(const std::string&name, int32_t data, int splen, int index);
void convert(const std::string&name, uint32_t data, int splen, int index);
void convert(const std::string&name, int64_t data, int splen, int index);
void convert(const std::string&name, uint64_t data, int splen, int index);
void convert(const std::string&name, float data, int splen, int index);
void convert(const std::string&name, double data, int splen, int index);
void convert(const std::string&name, bool data, int splen, int index);
// base type vector
void convert(const std::string&name, const std::vector<int16_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<uint16_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<int32_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<uint32_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<int64_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<uint64_t>& data, int splen, int index);
void convert(const std::string&name, const std::vector<float>& data, int splen, int index);
void convert(const std::string&name, const std::vector<double>& data, int splen, int index);
void convert(const std::string&name, const std::vector<bool>& data, int splen, int index);
void convert(const std::string&name, const std::vector<std::string>& data, int splen, int index);
// base type set
void convert(const std::string&name, const std::set<int16_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<uint16_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<int32_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<uint32_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<int64_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<uint64_t>& data, int splen, int index);
void convert(const std::string&name, const std::set<float>& data, int splen, int index);
void convert(const std::string&name, const std::set<double>& data, int splen, int index);
void convert(const std::string&name, const std::set<bool>& data, int splen, int index);
void convert(const std::string&name, const std::set<std::string>& data, int splen, int index);
template <typename TYPE>
void convert(const std::string&name, const std::vector<TYPE> &data, int splen, int index) {
linefeed(index);
std::string oldsep(_sep);
//_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
} else {
_ss<<std::string(splen, ' ')<<_vecb;
}
for (size_t i=0; i<data.size(); ++i) {
_sep=",";
this->convert("", data[i], splen+space(), (int)i);
_sep=oldsep;
}
_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<_vece;
}
template <typename TYPE>
void convert(const std::string&name, const std::set<TYPE> &data, int splen, int index) {
int i = 0;
linefeed(index);
std::string oldsep(_sep);
//_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
} else {
_ss<<std::string(splen, ' ')<<_vecb;
}
for (typename std::set<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {
_sep=",";
this->convert("", *it, splen+space(), i++);
_sep=oldsep;
}
_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<_vece;
}
template <typename TYPE>
void convert(const std::string&name, const std::map<std::string,TYPE> &data, int splen, int index) {
int i = 0;
linefeed(index);
std::string oldsep(_sep);
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<"{";
} else {
_ss<<std::string(splen, ' ')<<"{";
}
for (typename std::map<std::string,TYPE>::const_iterator iter=data.begin(); iter!=data.end(); ++iter) {
_sep=",";
this->convert(iter->first, iter->second, splen+space(), i++);
_sep=oldsep;
}
_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<"}";
}
template <typename KEYTYPE, typename TYPE>
void convert(const std::string&name, const std::map<KEYTYPE, TYPE> &data, int splen, int index) {
int i = 0;
linefeed(index);
std::string oldsep(_sep);
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<"{";
} else {
_ss<<std::string(splen, ' ')<<"{";
}
for (typename std::map<KEYTYPE,TYPE>::const_iterator iter=data.begin(); iter!=data.end(); ++iter) {
_sep=",";
std::string _k = boost::lexical_cast<std::string>(iter->first);
this->convert(_k, iter->second, splen+space(), i++);
_sep=oldsep;
}
_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<"}";
}
template <typename TYPE>
void convert(const std::string&name, const TYPE& data, int splen, int index) {
linefeed(index);
data.__struct_to_str(*this, name, splen);
}
private:
void linefeed(int index) {
if (index>0) {
_ss<<_sep;
}
if (_newline) {
_ss<<'\n';
}
}
template <typename TYPE>
void base_convert(const std::string&name, const TYPE& data, int splen, int index) {
linefeed(index);
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<data;
} else {
_ss<<std::string(splen, ' ')<<data;
}
}
template <typename TYPE>
void base_vector(const std::string&name, const std::vector<TYPE>& data, int splen, int index) {
linefeed(index);
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
} else {
_ss<<std::string(splen, ' ')<<_vecb;
}
for (typename std::vector<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {
if (it != data.begin()) {
_ss<<',';
}
_ss<<*it;
}
_ss<<_vece;
}
template <typename TYPE>
void base_set(const std::string&name, const std::set<TYPE>& data, int splen, int index) {
linefeed(index);
if (!name.empty()) {
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
} else {
_ss<<std::string(splen, ' ')<<_vecb;
}
for (typename std::set<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {
if (it != data.begin()) {
_ss<<',';
}
_ss<<*it;
}
_ss<<_vece;
}
private:
bool _json;
bool _newline;
int _space;
std::string _vecb; // json:[, cfg:(
std::string _vece; // json:], cfg:)
std::string _key; // json:", cfg:(empty)
std::string _assign;// json::, cfg:=
std::string _sep; // json:,, cfg:;
std::stringstream _ss;
};
}
#endif
jsonstrcfg.cpp json序列化反序列化定义
* Copyright (C) 2017 YY Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <json/json.h>
#include "jsoncfgstr.hpp"
namespace x2struct {
using namespace std;
JsonCfgStr::JsonCfgStr(bool isjson, bool newline, int splen)
:XStr(isjson?"json":"config"),_json(isjson), _newline(newline),_space(splen)
{
if (isjson) {
_vecb = "[";
_vece = "]";
_key = "\"";
_assign = ":";
_sep = ",";
} else {
_vecb = "(";
_vece = ")";
_assign = "=";
_sep = ";";
}
}
JsonCfgStr::~JsonCfgStr()
{
}
void JsonCfgStr::begin(const string&root, int splen)
{
if (root.empty()) {
_ss<<string(splen, ' ')<<"{";
} else {
_ss<<string(splen, ' ')<<_key<<root<<_key<<_assign<<"{";
}
}
void JsonCfgStr::end(const string&root, int splen)
{
if (_newline) {
_ss<<'\n';
}
_ss<<string(splen, ' ')<<"}";
}
int JsonCfgStr::space()
{
return _newline?_space:0;
}
string JsonCfgStr::toStr()const
{
return _ss.str();
}
void JsonCfgStr::convert(const string&name, int16_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const string&name, uint16_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const string&name, int32_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const string&name, uint32_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, int64_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, uint64_t data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, float data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, double data, int splen, int index)
{
base_convert(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, bool data, int splen, int index)
{
base_convert(name, data?"true":"false", splen, index);
}
void JsonCfgStr::convert(const string&name, const string& data, int splen, int index)
{
linefeed(index);
if (_json) {
_ss<<string(splen, ' ')<<_key<<name<<_key<<_assign<<Json::valueToQuotedString(data.c_str());
} else {
_ss<<string(splen, ' ')<<_key<<name<<_key<<_assign<<'"'<<data<<'"';
}
}
// base type vector
void JsonCfgStr::convert(const std::string&name, const std::vector<int16_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<uint16_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<int32_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<uint32_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<int64_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<uint64_t>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<float>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<double>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<bool>& data, int splen, int index)
{
base_vector(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::vector<std::string>& data, int splen, int index)
{
linefeed(index);
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
for (typename std::vector<std::string>::const_iterator it=data.begin(); it!=data.end(); ++it) {
if (it != data.begin()) {
_ss<<',';
}
_ss<<'"'<<*it<<'"';
}
_ss<<_vece;
}
// base type set
void JsonCfgStr::convert(const std::string&name, const std::set<int16_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<uint16_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<int32_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<uint32_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<int64_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<uint64_t>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<float>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<double>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<bool>& data, int splen, int index)
{
base_set(name, data, splen, index);
}
void JsonCfgStr::convert(const std::string&name, const std::set<std::string>& data, int splen, int index)
{
linefeed(index);
_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;
for (typename std::set<std::string>::const_iterator it=data.begin(); it!=data.end(); ++it) {
if (it != data.begin()) {
_ss<<',';
}
_ss<<'"'<<*it<<'"';
}
_ss<<_vece;
}
}
第二种。实际上没做序列化定义，支撑任意类型的快速写入和取出，尤其应用在于单生产者、单消费者线程的无锁共享。怎么把它改造成rpc形式的序列化和反序列化，甚至支撑任意类型存储。可以使用avro特化函数。也可以借用redis队列协议即（数据类型+数据）。只是给个引子，怎么实现各位可以自己思考
.h
#ifndef __LOGBLOCKMANER_H__
#define __LOGBLOCKMANER_H__
#include <stdio.h>
class LogBlockManer
{
public:
~LogBlockManer()
{
}
LogBlockManer()
{
_blockSize = 1;
}
private:
struct __BlockNode
{
int seq;
int w_offset;
int r_offset;
int memsize;
char * memory;
struct __BlockNode * next;
__BlockNode()
{
seq = 0;
w_offset = 0;
r_offset = 0;
memsize = 0;
memory = NULL;
next = NULL;
}
void reset(int size)
{
w_offset = 0;
r_offset = 0;
memsize = size;
next = NULL;
return ;
}
};
__BlockNode * _wrnode;
__BlockNode * _rdnode;
__BlockNode * _head;
int _blockSize;
int _threadmem;
public:
int Init();
int writeLog(char * log,int size,__FileInfo *&file);
bool readLog(char *& log,int &size,__FileInfo *&file);
int setThreadmemsize(int size);
};
#endif
.cpp#include "LogBlockManer.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MEMSIZE 1024*1024
int LogBlockManer::writeLog(char * log,int size,__FileInfo *&file)
{
static int int_size = sizeof(int);
static int charpoint_size = sizeof(char*);
if(_wrnode->w_offset + size + charpoint_size + int_size>= _wrnode->memsize)
{
if(!_wrnode->memory)
{
_wrnode->memory = new char[MEMSIZE];
if(!_wrnode->memory)
{
return 0;
}
_wrnode->memsize = MEMSIZE;
}else
{
__BlockNode * node = new __BlockNode();
if(!node)
{
return 0;
}
node->memory = new char[MEMSIZE];
if(!node->memory)
{
delete node;
return 1;
}
_blockSize++;
_wrnode->next = node;
_wrnode = node;
_wrnode->memsize = MEMSIZE;
}
}
}
memcpy((void*)(_wrnode->memory+_wrnode->w_offset),(void*)&file,charpoint_size);
memcpy((void*)(_wrnode->memory+_wrnode->w_offset+charpoint_size),(void*)&size,int_size);
memcpy((void*)(_wrnode->memory+_wrnode->w_offset+(charpoint_size+int_size)),(void*)log,size);
_wrnode->w_offset += (size+int_size+charpoint_size);
return 0;
}
bool LogBlockManer::readLog(char * & log,int & size,__FileInfo *&file)
{
static int int_size2 = sizeof(int);
static int charpoint_size = sizeof(char*);
if(_rdnode->r_offset >= _rdnode->w_offset)
{
if(_rdnode != _wrnode)
{
__BlockNode * node = _rdnode;
_rdnode = _rdnode->next;
delete [] node->memory;
delete node;
if(_rdnode->r_offset >= _rdnode->w_offset)
{
return false;
}
}else
{
return false;
}
}
memcpy((void*)&file,(void*)(_rdnode->memory + _rdnode->r_offset),charpoint_size);
memcpy((void*)&size,(void*)(_rdnode->memory + _rdnode->r_offset + charpoint_size),int_size2);
log = _rdnode->memory + (charpoint_size + int_size2+_rdnode->r_offset);
_rdnode->r_offset += (size+charpoint_size+int_size2);
return true;
}
int LogBlockManer::Init()
{
_wrnode = new __BlockNode();
if(!_wrnode)
{
return -1;
}
_head = _wrnode;
_rdnode = _wrnode;
return 0;
}
int LogBlockManer::setThreadmemsize(int size)
{
_threadmem = size;
return 0;
}
4810
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
