golang-thrift 使用hbase教程

1.hbase知识

	进入shell命令: /usr/local/hbase/bin/hbase shell 
	create 'ulive_test', 'i'  // 创建一个列族 i
	help // 查看所有命令
	list // 查看所有表
	describe 'student' // 查看表结构
	// 删除表之前需要 disable 
	disable 'student'
	drop 'student'
	命名空间
	list_namespace 
	创建空间
	create_namespace 'gitxuzan'
	create "gitxuzan:statue" ,'info'
	disable "gitxuzan:statue"
	drop "gitxuzan:stu"
	drop_namespace 'gitxuzan'
	put 'sut' ,'1001', 'info1:name', 'zhangsan'  //新增数据1001是rowkey
	get 'stu','1001' // 查询

更多命令相关使用查阅

2.Go使用Thrift连接HBase(🆚resetful api接口效率要高,相关开源包:https://github.com/tsuna/gohbase)

mac 通过 brew安装，windows 系统下载 thrift.exe 可执行文件 (windows下载链接:http://archive.apache.org/dist/thrift/)

 brew install thrift
 查看版本 thrift -version (注意版本号最好是能在github.com/apache/thrift 找到对应tag)
	Thrift version 0.14.1

3.下载HBase源码

https://hbase.apache.org/downloads.html

解压后来到 hbase-thrift\src\main\resources\org\apache\hadoop\hbase\thrift\Hbase.thrift 
执行： thrift -gen go -o . ./Hbase.thrift
生成 gen-go 放到你的项目,注意里面可能有string 类型的语法错误,需要自己替换

4.go 源码部署

注意：thrift 插件和你使用的版本 https://github.com/apache/thrift 需要一致，我这边的代码是使用的thrift v0.14.1版本

一个链接和插入的例子(表要先创建)

package hbases

import (
	"context"
	"github.com/Xuzan9396/zlog"
	"github.com/apache/thrift/lib/go/thrift"
	"github.com/panjf2000/ants/v2"
	"github.com/spf13/viper"
	"myhbase/common"
	"myhbase/global"
	"myhbase/gen-go/hbase"
	"net"
	"strconv"
	"sync"
	"time"
)

type NewHbase struct {
	data  chan []*common.UserEventInfo
	onces sync.Once
}

var HBASE *NewHbase

func GetModel() *NewHbase {
	if HBASE == nil {
		// 启动hbase
		model := &NewHbase{
			data: make(chan []*common.UserEventInfo, 2000),
		}
		HBASE = model
		model.cient()
		go model.HbasePull()
		go model.ping_cient()
	}

	return HBASE
}

func (c *NewHbase) cient() {
	host, portStr, table := viper.GetString("hbase.host"), viper.GetString("hbase.port"), viper.GetString("hbase.table")

	var err error
	// -------------------------    -----------------------
	protocolFactory := thrift.NewTBinaryProtocolFactoryConf(&thrift.TConfiguration{
		        TBinaryStrictRead:  thrift.BoolPtr(true),
		        TBinaryStrictWrite: thrift.BoolPtr(true),
		     })
	zlog.F().Info(host," ",portStr)
	global.Transport, err = thrift.NewTSocketConf(net.JoinHostPort(host, portStr),&thrift.TConfiguration{

		ConnectTimeout: time.Second*60,
		 SocketTimeout:  time.Second*60,
		     })
	if err != nil {
		zlog.F().Error(err)
		return
	}


	client := hbase.NewHbaseClientFactory(global.Transport, protocolFactory)
	if err := global.Transport.Open(); err != nil {
		zlog.F().Error(err)
		return
	}
	global.Hbase = client


	argvalue0 := []byte(table)
	value0 := hbase.Bytes(argvalue0)
	bools, err := client.IsTableEnabled(context.TODO(),value0)
	if err != nil {
		zlog.F().Error(err)
	}
	if bools == true {
		zlog.F().Info("表已存在:" + table)
	}

}

func (c *NewHbase) ping_cient() {
	defer func() {
		if r := recover(); r != nil {
			zlog.F().Error(r)
		}
	}()
	times := time.NewTicker(5 * time.Second)
	var err error
	for {
		select {
		case <-times.C:

			if global.Transport != nil && global.Transport.IsOpen() {
				zlog.F().Info("已经isopen")
				continue
			}

			zlog.F().Info("hbase断开链接,准备正在重连!")
			//global.LOG.Info("test!")
			host, portStr := viper.GetString("hbase.host"), viper.GetString("hbase.port")
			// -------------------------    -----------------------
			protocolFactory := thrift.NewTBinaryProtocolFactoryConf(&thrift.TConfiguration{
				TBinaryStrictRead:  thrift.BoolPtr(true),
				TBinaryStrictWrite: thrift.BoolPtr(true),
			})
			zlog.F().Info(host," ",portStr)
			global.Transport, err = thrift.NewTSocketConf(net.JoinHostPort(host, portStr),&thrift.TConfiguration{
				ConnectTimeout: time.Second*60,
				SocketTimeout:  time.Second*60,
			})
			if err != nil {
				zlog.F().Error(err)
				return
			}

			client := hbase.NewHbaseClientFactory(global.Transport, protocolFactory)
			if err := global.Transport.Open(); err != nil {
				zlog.F().Error(err)

				return
			}
			global.Hbase = client
			zlog.F().Info("重新连接池状态正常")
		}
	}

}

func (c *NewHbase) HbasePull() {
	defer func() {
		if r := recover(); r != nil {
			//global.LOG.Error("err", zap.Any("错误pull", r))
			zlog.F().Error(r)
		}
	}()
	p, _ := ants.NewPoolWithFunc(1, func(i interface{}) {
		data, ok := i.([]*common.UserEventInfo)
		if !ok {
			return
		}

		c.Put(data)

	})
	defer p.Release()
	for {
		select {
		case r := <-c.data:

			_ = p.Invoke(r)

		}
	}

}

func (c *NewHbase) Set(data []*common.UserEventInfo) {
	select {
	case c.data <- data:
	default:
		//global.LOG.Info("丢弃了")
		zlog.F().Info("丢弃了")

	}
}

func (c *NewHbase) Put(data []*common.UserEventInfo) {
	tableName, col := viper.GetString("hbase.table"), "i"
	//col := "i"
	sendMuData := make([]*hbase.BatchMutation, 0)
	for _, value := range data {
		if value.UserId <= 0 {
			continue
		}

		userIdStr := strconv.FormatInt(value.UserId, 10)
		timeStr := strconv.FormatInt(value.Time, 10)
		if len(timeStr) == 13 {
			timeStr = timeStr[:10]
		}
		widthStr := strconv.Itoa(value.Properties.ScreenWidth)
		heightStr := strconv.Itoa(value.Properties.ScreenHeight)

		rowKey := userIdStr + timeStr
		//m := hbase.Mutation{false, []byte(col + ":name"), []byte("xuzan2"), true}
		sendArr := []*hbase.Mutation{
			{false, []byte(col + ":user_id"), []byte(userIdStr), true},
			{false, []byte(col + ":event"), []byte(value.Event), true},
			{false, []byte(col + ":device_id"), []byte(value.DeviceID), true},
			{false, []byte(col + ":time"), []byte(timeStr), true},
			{false, []byte(col + ":lib"), []byte(value.Properties.Lib), true},
			{false, []byte(col + ":lib_version"), []byte(value.Properties.LibVersion), true},
			{false, []byte(col + ":model"), []byte(value.Properties.Model), true},
			{false, []byte(col + ":os"), []byte(value.Properties.Os), true},
			{false, []byte(col + ":screen_width"), []byte(widthStr), true},
			{false, []byte(col + ":screen_height"), []byte(heightStr), true},
			{false, []byte(col + ":manufacturer"), []byte(value.Properties.Manufacturer), true},
			{false, []byte(col + ":app_version"), []byte(value.Properties.AppVersion), true},
			{false, []byte(col + ":element_id"), []byte(value.Properties.ElementID), true},
			{false, []byte(col + ":activity"), []byte(value.Properties.Activity), true},
		}

		sendMuData = append(sendMuData, &hbase.BatchMutation{
			[]byte(rowKey), sendArr,
		})

	}

	if len(sendMuData) > 0 {

		err :=  global.Hbase.MutateRows(
			context.TODO(),
			[]byte(tableName),
			sendMuData,
			nil)
		if err == nil {
			zlog.F().Info("批量插入成功")
		} else {
			zlog.F().Error(err)
		}
	}
}

5.遇到的bug

bug1: "err": "mutateRows failed: out of sequence response
看这个问题，处理方式是减少了并发请求了，之前是开了5个线程,一个连接去请求的，改成一个线程就好了,如果需要并发，可能需要多个连接 (相关原因：https://bbs.huaweicloud.com/blogs/197750)
bug2: : write: broken pip  ，服务端修改了connect时间，服务器改了hbase配置hbase.thrift.server.socket.read.timeout (https://blog.csdn.net/u011342882/article/details/86300625)

6.其他相关链接

https://www.daryl.top/2021/04/go%E4%BD%BF%E7%94%A8thrift%E8%BF%9E%E6%8E%A5hbase/

https://blog.csdn.net/lesorb/article/details/64442673