Protobuf 总结

Protobuf 总结

本文索引

• Protobuf简介
• Python protobuf环境设置
• C# protobuf环境设置
• 注意事项

Protobuf简介

protocolbuffer(以下简称PB)是google 的一种数据交换的格式，它独立于语言，独立于平台。google 提供了多种语言的实现：java、c#、c++、go 和 python，每一种实现都包含了相应语言的编译器以及库文件。由于它是一种二进制的格式，比使用 xml 进行数据交换快许多。可以把它用于分布式应用之间的数据通信或者异构环境下的数据交换。作为一种效率和兼容性都很优秀的二进制数据传输格式，可以用于诸如网络传输、配置文件、数据存储等诸多领域。 

Python protobuf环境设置

1）安装protobuf 数据编译环境

apt-get install protobuf-compiler 

2）安装python protobuf解析环境

pip install protobuf

3）新建一个Data.proto(数据结构体)

PS:其中赋值部分相当于序号作用

package ProtoData;
//Public Part Start
message Proto_Vector3{
    optional float x=1;
    optional float y=2;
    optional float z=3;
}

message Proto_Quaternion{
    optional float x=1;
    optional float y=2;
    optional float z=3;
    optional float w=4;
}
//Public Part End

message Proto_CloneData{
    optional int32 userID=1;
    optional float sendTime=2;
    optional Proto_Vector3 pos=3;
    optional Proto_Quaternion rot=4;
    optional float engineVolume=5;
    optional float enginePitch=6;
    optional bool inSkid=7;
    optional float fWheelSteerAngle=8;
    optional float rWheelRpm=9;
    optional int32 finishCircleCount=10;
}

4）生成结构体python可调用的代码

protoc -I=./ --python_out=./ Data.proto

5）使用生成的代码进行编码/解码

import Data_pb2

class Vector3Data:
    __proto = None

    def __init__(self,x=0,y=0,z=0):
        self.__proto = Data_pb2.Proto_Vector3()
        self.__proto.x = x
        self.__proto.y = y
        self.__proto.z = z
    
    def getProto(self):
        return self.__proto

class QuaternionData:
    __proto = None

    def __init__(self,x=0,y=0,z=0,w=0):
        self.__proto = Data_pb2.Proto_Quaternion()
        self.__proto.x = x
        self.__proto.y = y
        self.__proto.z = z
        self.__proto.w = w
    
    def getProto(self):
        return self.__proto

class CloneData:
    __proto = None
    
    def __init__(self):
        self.__proto = Data_pb2.Proto_CloneData()

    def encode(self,user_id,send_time,pos,rot,engine_volume,engine_pitch,in_skid,fwheel_steer_angle,rwheel_rpm,finish_circle_count):
        self.__proto.userID = user_id
        self.__proto.sendTime = send_time
        self.__proto.pos.x = pos.getProto().x
        self.__proto.pos.y = pos.getProto().y
        self.__proto.pos.z = pos.getProto().z

        self.__proto.rot.x = rot.getProto().x
        self.__proto.rot.y = rot.getProto().y
        self.__proto.rot.z = rot.getProto().z
        self.__proto.rot.w = rot.getProto().w

        self.__proto.engineVolume = engine_volume
        self.__proto.enginePitch = engine_pitch
        self.__proto.inSkid = in_skid
        self.__proto.fWheelSteerAngle = fwheel_steer_angle
        self.__proto.rWheelRpm = rwheel_rpm
        self.__proto.finishCircleCount = finish_circle_count

        return self.__proto.SerializeToString()

    def decode(self,proto_str):
        self.__proto.ParseFromString(proto_str)
        return {
                "userID":self.__proto.userID,
                "sendTime":self.__proto.sendTime,
                "pos":{
                    "x":self.__proto.pos.x,
                    "y":self.__proto.pos.y,
                    "z":self.__proto.pos.z
                },
                "rot":{
                    "x":self.__proto.rot.x,
                    "y":self.__proto.rot.y,
                    "z":self.__proto.rot.z,
                    "w":self.__proto.rot.w
                },
                "engineVolume":self.__proto.engineVolume,
                "enginePitch":self.__proto.enginePitch,
                "inSkid":self.__proto.inSkid,
                "fWheelSteerAngle":self.__proto.fWheelSteerAngle,
                "rWheelRpm":self.__proto.rWheelRpm,
                "finishCircleCount":self.__proto.finishCircleCount
        }

'''def main():
    vector3Data = Vector3Data(1,2,3)
    quaternionData = QuaternionData(5,6,7)

    cloneData = CloneData()

def __synchronizePlayerInfo(self,data):
        proto_data = data["CloneData"]
        clone_proto_data = CloneData()
        clone_data = clone_proto_data.decode(base64.b64decode(proto_data))

        user_id = clone_data["userID"]
        send_time = clone_data["sendTime"]

C# protobuf环境设置

1）准备protobuf 数据编译环境

下载链接：链接：http://pan.baidu.com/s/1c4HVHW 密码：5y1b 

protogen目录为编译.proto文件到.cs

full目录下为各种环境的插件文件

2）生成结构体C#可调用的代码

protogen -i:..protoData.proto -protoData.cs

3）C#调用生成的代码

using UnityEngine;
using System;
using System.IO;
using ProtoBuf;
using ProtoData;
using LitJson;

[Serializable]
public class CloneData{
    public float sendTime;
    public int userID, finishCircleCount;
    public Vector3 pos;
    public Quaternion rot;
    public float engineVolume;
    public float enginePitch;
    public bool inSkid;
    public float FWheelSteerAngle;
    public float RWheelRpm;
    public bool flag;
    public static int dtime=1;
    public static int sendPackage = 0;
    public static int recvPackage = 0;

    public CloneData()
    {

    }

    public CloneData(string protoStr)
    {
        if (protoStr != null)
        {
            try
            {
                //对字符串进行base64解密为二进制数据
                byte[] byteArray = Convert.FromBase64String(protoStr)
                //反序列化
                MemoryStream ms = new MemoryStream(byteArray)
                Proto_CloneData protoCloneData = Serializer.Deserialize<Proto_CloneData>(ms)

                //从protobuf数据中提取数据
                this.sendTime = protoCloneData.sendTime;
                this.userID = protoCloneData.userID;
                this.pos = new Vector3(protoCloneData.pos.x, protoCloneData.pos.y, protoCloneData.pos.z)
                this.rot = new Quaternion(protoCloneData.rot.x, protoCloneData.rot.y, protoCloneData.rot.z, protoCloneData.rot.w)
                this.engineVolume = protoCloneData.engineVolume;
                this.enginePitch = protoCloneData.enginePitch;
                this.inSkid = protoCloneData.inSkid;
                this.FWheelSteerAngle = protoCloneData.fWheelSteerAngle;
                this.RWheelRpm = protoCloneData.rWheelRpm;
                this.finishCircleCount = protoCloneData.finishCircleCount;
            }
            catch
            {
                //数据解析异常
                this.flag = false;
                return;
            }
            //数据正常解析
            this.flag = true;
        }
    }

    /// <summary>
    /// 转化为Json数据
    /// </summary>
    /// <returns>JsonData数据</returns>
    public JsonData toJson()
    {
        float timeStamp = Time.time;
        JsonData jsonData = new JsonData()
        //将数据保存为protobuf格式
        Proto_CloneData proto_CloneData = new Proto_CloneData()

        proto_CloneData.sendTime = timeStamp;
        proto_CloneData.userID = this.userID;

        proto_CloneData.pos = new Proto_Vector3()
        proto_CloneData.pos.x = this.pos.x;
        proto_CloneData.pos.y = this.pos.y;
        proto_CloneData.pos.z = this.pos.z;

        proto_CloneData.rot = new Proto_Quaternion()
        proto_CloneData.rot.x = this.rot.x;
        proto_CloneData.rot.y = this.rot.y;
        proto_CloneData.rot.z = this.rot.z;
        proto_CloneData.rot.w = this.rot.w;

        proto_CloneData.engineVolume = this.engineVolume;
        proto_CloneData.enginePitch = this.enginePitch;
        proto_CloneData.inSkid = this.inSkid;
        proto_CloneData.fWheelSteerAngle = this.FWheelSteerAngle;
        proto_CloneData.rWheelRpm = this.RWheelRpm;
        proto_CloneData.finishCircleCount = this.finishCircleCount;

        //protobuf序列化
        MemoryStream ms = new MemoryStream()
        Serializer.Serialize<Proto_CloneData>(ms, proto_CloneData)
        byte[] byteData = ms.ToArray()
        //将二进制数据进行base64加密为文本
        string data = Convert.ToBase64String(byteData, 0, byteData.Length)

        //保存到json中
        jsonData["CloneData"] = data;
        jsonData["userID"] = proto_CloneData.userID;
        jsonData["sendTime"] = timeStamp;
        return jsonData;
    }
}

Js Protobuf

Js本身不支持 序列化/反序列化 二进制，如果要使用Protobuf的话，需要借助ProtobufJs解决。

ProtobufJS:  https://www.npmjs.com/package/protobufjs 

Demo下载： http://www.linguang.space/file/upload/2017-09-14/15053185834009.7z

Protobuf基类

require("lib/ByteBufferAB.min.js");
require("lib/Long.min.js");
require("lib/ProtoBuf.min.js");

/**
 * protobuf数据对象基类
 */
export class Protobuf{
    private root;//protobuf根节点
    private typeID;//protobuf ID，用于对象池
    protected protobufKeyValue;//protobuf键值对
    protected protobufClass;//protobuf类
    protected protobufObject;//protobuf实例化对象

    /**
     * 构造函数
     * @param typeID protobufID，用于对象池
     * @param moduleName proto文件中message类名
     */
    constructor(typeID:number,moduleName:string){
        let protoData = `
        package protobuf;
        
        message LoginRequestMessage{
            required string userName = 1;
            required int32 userPasswd = 2;
        }
        `
        this.typeID = typeID;
        this.protobufKeyValue = {};
        this.root = protobuf.parse(protoData, { keepCase: true }).root;
        this.protobufClass = this.root.lookup(moduleName);
    }
    
    /**
     * 设置键值
     * @param keyValueArray 
     */
    protected _set(protobufDataKV:any){
        this.protobufObject =  this.protobufClass.create(protobufDataKV);
    }

    /**
     * 序列化
     */
    protected _encode(){
        //通过protobuf类
        return this.protobufClass.encode(this.protobufObject).finish();
    }

    /**
     * 反序列化
     */
    protected _decode(arrayBuffer:any){
        //注意:
        //我使用Websocket的传输格式为 arrayBuffer
        //此处需要将ArrayBuffer转化为Uint8Array再进行解析
        return this.protobufClass.decode(new Uint8Array(arrayBuffer));
    }

    /**
     * 获取protobuf类型ID
     */
    public getTypeID():number{
        return this.typeID;
    }
}

Protobuf对象池，用于降低GC

import {Protobuf} from "./Protobuf";
import {LoginRequestProtobuf} from "./LoginRequestProtobuf";

/**
 * protobuf数据类型
 */
export enum ProtobufDataType{
    LoginRequestMessage = 1,
}

/**
 * protobuf数据对象池
 */
export class ProtobufPool{
    public static instance:ProtobufPool;
    //protobuf对象类
    public static protobufClasses = {
        1:LoginRequestProtobuf,
    }

    public protobufs:any;

    constructor(){
        this.protobufs = {};
    }

    /**
     * 单例对象
     */
    public static getInstance():ProtobufPool{
        if (!this.instance){
            this.instance = new ProtobufPool();
        }

        return this.instance;
    }

    /**
     * 获取protobuf数据对象
     * @param protobufType protobuf数据对象类型
     */
    public static Get(protobufType:ProtobufDataType):any{
        if (this.getInstance().protobufs[protobufType]){
            //存在protobuf数据对象
            return this.getInstance().protobufs[protobufType].pop();
        }else{
            //不存在，创建新的数据对象
            return new ProtobufPool.protobufClasses[protobufType]();
        }
    }

    /**
     * 将protobuf数据对象放入对象池
     * @param protobuf 
     */
    public static Put(protobuf:Protobuf){
        if (!this.getInstance().protobufs[protobuf.getTypeID()]){
            //该类型protobuf还没有集合，创建一个
            this.getInstance().protobufs[protobuf.getTypeID()] = [];
        }

        this.getInstance().protobufs[protobuf.getTypeID()].push(protobuf);
    }
}

Protobuf子类，自定义扩展用

import {Protobuf} from "./Protobuf"
import {ProtobufDataType} from "./ProtobufPool";

/**
 * 登陆 protobuf属性,注意此处""中的内容必须与proto文件中到一致
 */
export enum LoginRequestProtobufAttr{
    UserName = "userName",
    UserPasswd = "userPasswd",
}

/**
 * 登陆 protobuf类
 */
export class LoginRequestProtobuf extends Protobuf{
    public userName:string;
    public userPasswd:string;

    constructor(){
        super(ProtobufDataType.LoginRequestMessage,"TankMoveMessage");
    }

    /**
     * 设置值（序列化前）
     * @param userName 玩家名称
     * @param userPasswd 玩家密码
     */
    public set(userName:string,userPasswd:string){
        this.protobufKeyValue[LoginRequestProtobufAttr.UserName] = userName;
        this.protobufKeyValue[LoginRequestProtobufAttr.UserPasswd] = userPasswd;
        //设置到protobuf中
        super._set(this.protobufKeyValue);
    }

    /**
     * 序列化数据
     */
    public encode(){
        //序列化
        return super._encode();
    }

    /**
     * 反序列化
     * @param buf 数据
     */
    public decode(buf:any){
        //反序列化
        let data = super._decode(buf);
        //保存值到本对象中
        this.userName = data[LoginRequestProtobufAttr.UserName];
        this.userPasswd = data[LoginRequestProtobufAttr.UserPasswd];
    }
}

注意事项

1）protobuf生成的二进制数据最好使用base64转化为字符串进行传输，否则容易出现问题。

2）客户端和服务器端应该使用同个版本的protobuf编译器。