skynet 使用protobuf

一、安装protobuf

    下面的操作方法都是在 centos 环境下操作

#下载 Protocol Buffers 源代码：
#您可以从 Protocol Buffers 的 GitHub 仓库中获取特定版本的源代码。使用以下命令克隆仓库
git clone -b v3.20.3 https://github.com/protocolbuffers/protobuf.git

#编译和安装：
#进入克隆的目录，然后编译和安装 Protocol Buffers：
cd protobuf
./autogen.sh
./configure
make
sudo make install

#验证安装：
protoc --version

#您应该看到输出，指示安装的版本为 3.20.3。

二、安装pdb库

#第三方库安装在3rd目录下
cd skynet/3rd/
git clone https://github.com/cloudwu/pbc.git

cd pbc
make



#编译成功后，打开skynet/3rd/pbc/binding/lua53/Makefile文件，修改里面的lua路径
CC = gcc
  CFLAGS = -O2 -fPIC -Wall
  LUADIR = ../../../lua   #这个路劲就是skynet/3rd/lua
  TARGET = protobuf.so
  
  .PHONY : all clean
  
  all : $(TARGET)
  
  $(TARGET) : pbc-lua53.c
      $(CC) $(CFLAGS) -shared -o $@ -I../.. -I$(LUADIR) -L../../build $^ -lpbc
  
  clean :
      rm -f $(TARGET)


#去到lua53目录，编译生成protobuf.so库
cd ./binding/lua53
sudo make

三、将依赖文件放到工程目录下

   将 protobuf.so 和 protobuf.lua 分别放入 luaclib 、lualib

cp protobuf.so ../../../../luaclib/  #将protobuf.so复制到存放C模块的lualib目录中    
cp protobuf.lua ../../../../lualib/  #将protobuf.lua复制到存放Lua模块的lualib目录中

四、创建protobuf目录

在skynet目录下创建protobuf目录，用来存放原始 .proto 描述文件

  这里举例 login.proto

mkdir protobuf

login.proto的内容如下代码所示：
syntax = "proto3";  
package Login;

message login {
	string account=1;
	string passwd=2;
	int32 result=3;
}

五、编译proto文件

protoc --descriptor_set_out login.pb login.proto

   这里写了个gen_pb.sh脚本，将protobuf目录下的所有.sproto文件转成proto文件

#!/bin/bash  
  
# 指定要遍历的目录，这里使用当前目录"."  
dir="."  
  
# 使用find命令查找所有.proto文件，并调用basename和cut命令来截取文件名  
find "$dir" -type f -name "*.proto" | while read -r filepath; do  
    # 使用basename命令获取文件名部分，然后使用cut命令去除后缀  
    filename=$(basename "$filepath")  
    filename_without_extension="${filename%.*}"  
      
    # 输出截取后的文件名  
    echo "$filename, $filename_without_extension"  
      
    # 在这里可以添加其他操作，比如使用protoc编译等  
     protoc --descriptor_set_out "$filename_without_extension.pb" "$filename" 
done

六、了解protobuf.lua 关键函数

1、 protobuf.register

• 功能：注册 Protobuf 
• 参数：buffer  .pb文件读取出来的二进制字符串
• 返回值：无。

function M.register(buffer)
    c._env_register(P, buffer) 
end

   备注： register需要自己去加载.pb文件内容，下面的register_file函数使用会更多

2、protobuf.register_file

• 功能：注册 Protobuf 
• 参数：filename 为  .pb文件名。  
• 返回值：无。

function M.register_file(filename)
    local f = assert(io.open(filename , "rb"))
    local buffer = f:read "*a"
    c._env_register(P, buffer)
    f:close()
end

3、protobof.encode

• 功能：将一个 Lua 表编码为 Protobuf 格式的二进制消息。
• 参数：message 是注册的 Protobuf 定义的名称，msg 是要编码的 Lua 表。
• 返回值：编码后的二进制数据。

function M.encode( message, t , func , ...)
    local encoder = c._wmessage_new(P, message)
    assert(encoder ,  message)
    encode_message(encoder, message, t)
    if func then
        local buffer, len = c._wmessage_buffer(encoder)
        local ret = func(buffer, len, ...)
        c._wmessage_delete(encoder)
        return ret
    else    
        local s = c._wmessage_buffer_string(encoder)
        c._wmessage_delete(encoder)
        return s
    end
end

4、protobof.decode

• 功能：将一个 Protobuf 编码的二进制消息解码为 Lua 表。
• 参数：typename 是注册的 Protobuf 定义的名称，buf 是包含 Protobuf 编码消息的二进制数据。
• 返回值：解码后的 Lua 表。

function M.decode(typename, buffer, length)
    local ret = {}
    local ok = c._decode(P, decode_message_cb , ret , typename, buffer, length)
    if ok then
        return setmetatable(ret , default_table(typename))
    else
        return false , c._last_error(P)
    end
end

七、protobuf测试用例

   在examples 目录下新建 test_protobuf.lua 

package.path = package.path .. ";./lualib/?.lua"
package.cpath = package.cpath .. ";./luaclib/?.so"

local protobuf = require "protobuf"      --引入文件protobuf.lua
protobuf.register_file "./protobuf/common.pb" --注册pb文件
protobuf.register_file "./protobuf/login.pb" --注册pb文件



local loginInfo = { account = "test", passwd = "pw"} 

local encodeData = protobuf.encode("Login.login", loginInfo)
print("encodeData:", encodeData)

local decodeData = protobuf.decode("Login.login", encodeData)
print("decodeData account:", decodeData.account)
print("decodeData passwd:", decodeData.passwd)

八、skynet 使用protobuf进行网络通信

1、 将数据打包成二进制数据

--[[
    big endian
    head    
        2 byte body size
        2 byte protonamesize
        n byte protoname
    body    
        n byte data
    @desc: 将lua格式的协议序列化为二进制数据
]] 
function protobufDataHelper.encode( name,data )
    local stringbuffer =  protobuf.encode(name, data)         -- protobuf序列化 返回lua string
    local body = string.pack(">s2s",name,stringbuffer)       -- 打包包体 协议名 + 协议数据
    local head = string.pack(">I2",#body)                     -- 打包包体长度
    print("encode proto_name:", name, ",data_size:", #body, ",totalSize:", #head+#body)
    return head .. body                                       -- 包体长度 + 协议名 + 协议数据
end

2、将二进制数据解包

--[[
    @desc: 将二进制数据反序列化为lua string
    --@msg: C Point 
    @return:协议名字，协议数据
]]
function protobufDataHelper.decode( msg  )
    --- 前两个字节在netpack.filter 已经解析
    print("msg size:", #msg)
    local proto_name,stringbuffer = string.unpack(">s2s",msg)
    print("proto_name", proto_name, "data:", stringbuffer)
    local body = protobuf.decode(proto_name, stringbuffer)
    return proto_name,body
end

3、skyent unpack类型指定为二进制字符串

     这里在agent.lua 注册消息协议类型时处理

skynet.register_protocol {
    name = "client",
    id = skynet.PTYPE_CLIENT,   
    unpack = skynet.tostring,   --- 将C point 转换为lua 二进制字符串
}

  在dispatch_message消息地方反序列化消息

--- 分发消息
local function  dispatch_message(msg)
    --- 反序列化二进制string数据
    local pack_name,data = dataHelper.decode(msg)     --   pack_name = c2s.test
    local sub_name = pack_name:match(".+%.(%w+)$")    --   sub_name = test

    ......

    local f = REQUEST[sub_name]
    if f == nil then
        print("not function define handle package:", pack_name)
        return
    end
    f(data)
end

agent.lua 改造测试代码如下

local skynet = require "skynet"
local socket = require "skynet.socket"
local sproto = require "sproto"
local sprotoloader = require "sprotoloader"
local login = require "login"
local tableutil = require "tableutil"

local dataHelper = require "protobufDataHelper"

local WATCHDOG
local host
local send_request

local CMD = {}
local REQUEST = {}
local client_fd


function REQUEST:get()
	print("get", self.what)
	local r = skynet.call("SIMPLEDB", "lua", "get", self.what)
	return { result = r }
end

function REQUEST:set()
	print("set", self.what, self.value)
	local r = skynet.call("SIMPLEDB", "lua", "set", self.what, self.value)
end

function REQUEST:handshake()
	return { msg = "Welcome to skynet, I will send heartbeat every 5 sec." }
end

function REQUEST:quit()
	skynet.call(WATCHDOG, "lua", "close", client_fd)
end

local function send_data(name, args)
	local data = dataHelper.encode(name, args)
	-- 发送数据
	socket.write(client_fd,data)
end

function REQUEST:login()
	print("login account,passwd:", self.account, self.passwd)
	local result = login.loginRequest(self.account, self.passwd)
	if result ~= 0 then
		print("kill client, client_fd:", client_fd)
		REQUEST:quit()
	end

	local loginInfo = { account = "kk", passwd = "haha"}
	send_data("Login.login", loginInfo)
	
	return {result = result}
end

function REQUEST:loginTest()
	print("loginTest:", tableutil.tPrint(self))
end


local function request(name, args, response)
	local f = assert(REQUEST[name])
	print("recieve request, protoName:", name, tableutil.tPrint(args))
	local r = f(args)
	if response then
		return response(r)
	end
end

local function send_package(pack)
	local package = string.pack(">s2", pack)
	socket.write(client_fd, package)
end

--[[
skynet.register_protocol {
	name = "client",
	id = skynet.PTYPE_CLIENT,
	unpack = function (msg, sz)
		return host:dispatch(msg, sz)
	end,
	dispatch = function (fd, _, type, ...)
		assert(fd == client_fd)	-- You can use fd to reply message
		skynet.ignoreret()	-- session is fd, don't call skynet.ret
		skynet.trace()
		if type == "REQUEST" then
			local ok, result  = pcall(request, ...)
			if ok then
				if result then
					send_package(result)
				end
			else
				skynet.error(result)
			end
		else
			assert(type == "RESPONSE")
			error "This example doesn't support request client"
		end
	end
}--]]

skynet.register_protocol {
    name = "client",
    id = skynet.PTYPE_CLIENT,	
    unpack = skynet.tostring,   --- 将C point 转换为lua 二进制字符串
}

function CMD.start(conf)
	local fd = conf.client
	local gate = conf.gate
	WATCHDOG = conf.watchdog
	-- slot 1,2 set at main.lua
	host = sprotoloader.load(1):host "package"
	send_request = host:attach(sprotoloader.load(2))
	skynet.fork(function()
		local index = 1
		while true do
			
			--send_package(send_request "heartbeat")
			index = index + 1
			local loginInfo = { account = "kk"..index, passwd = "haha"}
			send_data("Login.login", loginInfo)
			skynet.sleep(500)
		end
	end)

	client_fd = fd
	skynet.call(gate, "lua", "forward", fd)
end

function CMD.disconnect()
	-- todo: do something before exit
	skynet.exit()
end

--- 分发消息
local function  dispatch_message(msg)
	--- 反序列化二进制string数据
	local pack_name,data = dataHelper.decode(msg)     --   pack_name = c2s.test
	local sub_name = pack_name:match(".+%.(%w+)$")    --   sub_name = test

	print("recieve request, protoName:", pack_name, tableutil.tPrint(data))
	
	local f = REQUEST[sub_name]
	if f == nil then
		print("not function define handle package:", pack_name)
		return
	end
	f(data)
end

skynet.start(function()
	skynet.dispatch("lua", function(_,_, command, ...)
		skynet.trace()
		local f = CMD[command]
		skynet.ret(skynet.pack(f(...)))
	end)
	skynet.dispatch("client", function (session, address, msg)
		dispatch_message(msg)
	end)
end)

4、客户端测试代码

client_protobuf.lua

package.cpath = "luaclib/?.so;skynet/luaclib/?.so"
package.path = "lualib/common/?.lua;lualib/?.lua;skynet/lualib/?.lua;skynet/examples/?.lua"

local tableutil = require "tableutil"

if _VERSION ~= "Lua 5.4" then
    error "Use lua 5.4"
end



local socket = require "client.socket"
local dataHelper = require "protobufDataHelper"

local fd = assert(socket.connect("127.0.0.1", 8888))


local function send_data(name, args)
    local data = dataHelper.encode(name, args)
    -- 发送数据
    socket.send(fd,data)
end

local loginInfo = { account = "test", passwd = "haha"}
send_data("Login.login", loginInfo)
send_data("Common.heartbeat", {})


local function unpack_package(text)
    local size = #text
    if size < 2 then
        return nil, text
    end
    local s = text:byte(1) * 256 + text:byte(2)
    if size < s+2 then
        return nil, text
    end

    return text:sub(3,2+s), text:sub(3+s)
end

local function recv_package(last)
    local result
    result, last = unpack_package(last)
    if result then
        return result, last
    end
    local r = socket.recv(fd)
    if not r then
        return nil, last
    end
    if r == "" then
        error "Server closed"
    end
    return unpack_package(last .. r)
end

local last = ""
local function dispatch_package()
    while true do
        local v
        v, last = recv_package(last)
        if not v then
            break
        end

	local packName, data = dataHelper.decode(v)
	print("packName:", packName)
        print("data:", tableutil.tPrint(data) )
    end
end

while true do

	dispatch_package()
	socket.usleep(1000000)
end