golang中使用etcd作为中间件踩坑

一、rpc error: code = ResourceExhausted desc = trying to send message larger than max (4194325 vs. 1048576)

原因分析：

初始化参数中未指定MaxCallSendMsgSize参数

func (ec *EtcdCliV3) Init(cfg *EtcdCliConf) (err error) {
	dialTimtout := cfg.DialTimeout
	if dialTimtout == 0 {
		dialTimtout = DEFAULT_DIAL_TIMOUT
	}
	etcdConfig := clientv3.Config{
		Endpoints:   cfg.Endpoints,
		DialTimeout: dialTimtout,
		Username:    cfg.Username,
		Password:    cfg.Password,
		DialOptions: []grpc.DialOption{grpc.WithBlock()},
	}
	if ec.client, err = clientv3.New(etcdConfig); err != nil {
		err = fmt.Errorf("init etcd cli fail, err: %v", err)
		return
	}
	return
}

etcd client初始化的时候使用了默认值 

if cfg.MaxCallSendMsgSize > 0 || cfg.MaxCallRecvMsgSize > 0 {
		if cfg.MaxCallRecvMsgSize > 0 && cfg.MaxCallSendMsgSize > cfg.MaxCallRecvMsgSize {
			return nil, fmt.Errorf("gRPC message recv limit (%d bytes) must be greater than send limit (%d bytes)", cfg.MaxCallRecvMsgSize, cfg.MaxCallSendMsgSize)
		}
		callOpts := []grpc.CallOption{
			defaultFailFast,
			defaultMaxCallSendMsgSize,
			defaultMaxCallRecvMsgSize,
		}
		if cfg.MaxCallSendMsgSize > 0 {
			callOpts[1] = grpc.MaxCallSendMsgSize(cfg.MaxCallSendMsgSize)
		}
		if cfg.MaxCallRecvMsgSize > 0 {
			callOpts[2] = grpc.MaxCallRecvMsgSize(cfg.MaxCallRecvMsgSize)
		}
		client.callOpts = callOpts
	}

 默认值 为2M

    // client-side request send limit, gRPC default is math.MaxInt32
	// Make sure that "client-side send limit < server-side default send/recv limit"
	// Same value as "embed.DefaultMaxRequestBytes" plus gRPC overhead bytes
	defaultMaxCallSendMsgSize = grpc.MaxCallSendMsgSize(2 * 1024 * 1024)

	// client-side response receive limit, gRPC default is 4MB
	// Make sure that "client-side receive limit >= server-side default send/recv limit"
	// because range response can easily exceed request send limits
	// Default to math.MaxInt32; writes exceeding server-side send limit fails anyway
	defaultMaxCallRecvMsgSize = grpc.MaxCallRecvMsgSize(math.MaxInt32)

解决方案：

//etcd客户端初始化
func (ec *EtcdCliV3) Init(cfg *EtcdCliConf) (err error) {
	dialTimtout := cfg.DialTimeout
	if dialTimtout == 0 {
		dialTimtout = DEFAULT_DIAL_TIMOUT
	}
	etcdConfig := clientv3.Config{
		Endpoints:   cfg.Endpoints,
		DialTimeout: dialTimtout,
		Username:    cfg.Username,
		Password:    cfg.Password,
		DialOptions: []grpc.DialOption{grpc.WithBlock()},
		MaxCallSendMsgSize:4 * 1024 * 1024,
	}
	if ec.client, err = clientv3.New(etcdConfig); err != nil {
		err = fmt.Errorf("init etcd cli fail, err: %v", err)
		return
	}
	return
}

注意：此外参数MaxCallRecvMsgSize也要特别注意，我现在还没遇坑。

二、code = ResourceExhausted desc = grpc: received message larger than max (4161420 vs. 2097152)

原因分析：etcd默认的grpc消息限制为2M

解决方案：在etcd的配置文件中加上如下配置限制，这里配的是4M

max-request-bytes: 4194304

注意：上述限制是为了保证etcd的性能而作出的限制，随着消息的增大，etcd的读写性能将会受到较大的影响；因此在遇到上述错误的时候，用户首先应该想着优化自己的方案使得etcd的key和value尽量小，而不是马上想着去改配置。

三、高并发写性能表现不好

原因分析：key或value较大导致的写缓慢。

解决方案：从业务角度出发，尽量避免大key或value，以此来提升高并发性能

四、高并发导致超出etcd集群承受范围导致的too many requests等问题

原因分析：再牛逼的集群，优化做的再好，也有承受临界

解决方案：做好度或写失败后的业务回滚