Kitex 探究服务注册

服务注册介绍

​ 微服务架构下，一个系统通常由多个微服务组成（比如电商系统可能分为用户服务，商品服务，订单服务等服务），一个用户请求可能会需要多个服务参与，这些服务之间互相配合以维持系统的正常运行。**服务注册（Service Registration）**是指将服务提供者的信息登记到一个注册中心的过程。这个过程在分布式系统和微服务架构中非常重要，因为它允许服务消费者动态地发现和访问服务提供者。

kitex中已经支持如下服务注册中心;

• etcd
• nacos
• zookeeper
• polaris
• eureka
• consul

等等

用法

使用nacos作为服务注册中心

registry-nacos/example/basic/server/main.go at main · kitex-contrib/registry-nacos (github.com)

// Copyright 2021 CloudWeGo Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"log"
	"net"

	"github.com/cloudwego/kitex/pkg/rpcinfo"
	"github.com/cloudwego/kitex/server"
	"github.com/kitex-contrib/registry-nacos/example/hello/kitex_gen/api"
	"github.com/kitex-contrib/registry-nacos/example/hello/kitex_gen/api/hello"
	"github.com/kitex-contrib/registry-nacos/registry"
)

type HelloImpl struct{}

func (h *HelloImpl) Echo(_ context.Context, req *api.Request) (resp *api.Response, err error) {
	resp = &api.Response{
		Message: req.Message,
	}
	return
}

func main() {
	r, err := registry.NewDefaultNacosRegistry()
	if err != nil {
		panic(err)
	}
	svr := hello.NewServer(
		new(HelloImpl),
		server.WithRegistry(r),
		server.WithServerBasicInfo(&rpcinfo.EndpointBasicInfo{ServiceName: "Hello"}),
		server.WithServiceAddr(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8080}),
	)
	if err := svr.Run(); err != nil {
		log.Println("server stopped with error:", err)
	} else {
		log.Println("server stopped")
	}
}

使用etcd作为服务注册中心

// Copyright 2021 CloudWeGo Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"log"
	"time"

	"github.com/cloudwego/kitex-examples/hello/kitex_gen/api"
	"github.com/cloudwego/kitex-examples/hello/kitex_gen/api/hello"
	"github.com/cloudwego/kitex/client"
	etcd "github.com/kitex-contrib/registry-etcd"
)

func main() {
	r, err := etcd.NewEtcdResolver([]string{"127.0.0.1:2379"})
	if err != nil {
		log.Fatal(err)
	}
	client := hello.MustNewClient("Hello", client.WithResolver(r))
	for {
		ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
		resp, err := client.Echo(ctx, &api.Request{Message: "Hello"})
		cancel()
		if err != nil {
			log.Fatal(err)
		}
		log.Println(resp)
		time.Sleep(time.Second)
	}
}

​ 可以看到，不同的服务注册中心使用方式基本一致，使用相当简单。

源码分析

kitex如何加载和启动配置中心

我们从注册位置开始：

svr := hello.NewServer(
		new(HelloImpl),
		server.WithRegistry(r),
		server.WithServerBasicInfo(&rpcinfo.EndpointBasicInfo{ServiceName: "Hello"}),
		server.WithServiceAddr(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8080}),

使用过kitex的开发者应该很熟悉（快速开始 | CloudWeGo），这行代码用于创建一个RPC的服务端服务，server.WithRegistry(r)这行代码用于注册配置中心。

服务注册client的配置

参数为服务注册的组件

// WithRegistry to set a Registry to register service
func WithRegistry(r registry.Registry) Option {
	return Option{F: func(o *internal_server.Options, di *utils.Slice) {
		di.Push(fmt.Sprintf("WithRegistry(%T)", r))
		o.Registry = r
	}}
}

参数为服务注册的组件, 显然参数为一个抽象的接口，不同的注册中心都可以用该接口表示，这也是之后服务注册中心的接口。

// Registry is extension interface of service registry.
type Registry interface {
	Register(info *Info) error
	Deregister(info *Info) error
}

函数最中把注册中心赋值：o.Registry = r

接着简单查看option

// Options is used to initialize the server.
type Options struct {
	Svr      *rpcinfo.EndpointBasicInfo
	Configs  rpcinfo.RPCConfig
	LockBits int
	Once     *configutil.OptionOnce

	MetaHandlers []remote.MetaHandler

	RemoteOpt  *remote.ServerOption
	ErrHandle  func(error) error
	ExitSignal func() <-chan error
	Proxy      proxy.ReverseProxy

	// Registry is used for service registry.
	Registry registry.Registry
	// RegistryInfo is used to in registry.
	RegistryInfo *registry.Info

	ACLRules []acl.RejectFunc
	Limit    Limit

	MWBs []endpoint.MiddlewareBuilder

	Bus    event.Bus
	Events event.Queue

	SupportedTransportsFunc func(option remote.ServerOption) []string

	// DebugInfo should only contain objects that are suitable for json serialization.
	DebugInfo    utils.Slice
	DebugService diagnosis.Service

	// Observability
	TracerCtl  *internal_stats.Controller
	StatsLevel *stats.Level
}

服务注册组件的使用

查看启动服务的代码

// Run runs the server.
func (s *server) Run() (err error) {
	s.Lock()
	s.isRun = true
	s.Unlock()
	if err = s.check(); err != nil {
		return err
	}
	diagnosis.RegisterProbeFunc(s.opt.DebugService, diagnosis.ServiceInfosKey, diagnosis.WrapAsProbeFunc(s.svcs.getSvcInfoMap()))
	svrCfg := s.opt.RemoteOpt
	addr := svrCfg.Address // should not be nil
	if s.opt.Proxy != nil {
		svrCfg.Address, err = s.opt.Proxy.Replace(addr)
		if err != nil {
			return
		}
	}

	s.fillMoreServiceInfo(s.opt.RemoteOpt.Address)
	s.richRemoteOption()
	transHdlr, err := s.newSvrTransHandler()
	if err != nil {
		return err
	}
	s.Lock()
	s.svr, err = remotesvr.NewServer(s.opt.RemoteOpt, s.eps, transHdlr)
	s.Unlock()
	if err != nil {
		return err
	}

	// start profiler
	if s.opt.RemoteOpt.Profiler != nil {
		gofunc.GoFunc(context.Background(), func() {
			klog.Info("KITEX: server starting profiler")
			err := s.opt.RemoteOpt.Profiler.Run(context.Background())
			if err != nil {
				klog.Errorf("KITEX: server started profiler error: error=%s", err.Error())
			}
		})
	}

	errCh := s.svr.Start()
	select {
	case err = <-errCh:
		klog.Errorf("KITEX: server start error: error=%s", err.Error())
		return err
	default:
	}
	muStartHooks.Lock()
	for i := range onServerStart {
		go onServerStart[i]()
	}
	muStartHooks.Unlock()
	s.Lock()
	s.buildRegistryInfo(s.svr.Address())
	s.Unlock()

	if err = s.waitExit(errCh); err != nil {
		klog.Errorf("KITEX: received error and exit: error=%s", err.Error())
	}
	if e := s.Stop(); e != nil && err == nil {
		err = e
		klog.Errorf("KITEX: stop server error: error=%s", e.Error())
	}
	return
}

其中调用了waitExit函数启动注册中心服务

func (s *server) waitExit(errCh chan error) error {
	exitSignal := s.opt.ExitSignal()

	// service may not be available as soon as startup.
	delayRegister := time.After(1 * time.Second)
	for {
		select {
		case err := <-exitSignal:
			return err
		case err := <-errCh:
			return err
		case <-delayRegister:
			s.Lock()
			if err := s.opt.Registry.Register(s.opt.RegistryInfo); err != nil {
				s.Unlock()
				return err
			}
			s.Unlock()
		}
	}
}

由此也就明了，在服务启动时，尝试启动服务后，会调用注册函数。因为整个链路都是接口，所以就能把关键逻辑抽离到具体的组件中单独维护，保证核心链路不变；只对可变部分扩展等。

registry-nacos 代码分析

接下来分析registry-nacos，注册client

r, err := registry.NewDefaultNacosRegistry()获取了nacos的client

// NewDefaultNacosResolver create a default service resolver using nacos.
func NewDefaultNacosResolver(opts ...Option) (discovery.Resolver, error) {
	cli, err := nacos.NewDefaultNacosClient()
	if err != nil {
		return nil, err
	}
	return NewNacosResolver(cli, opts...), nil
}

// NewNacosResolver create a service resolver using nacos.
func NewNacosResolver(cli naming_client.INamingClient, opts ...Option) discovery.Resolver {
	op := options{
		cluster: "DEFAULT",
		group:   "DEFAULT_GROUP",
	}
	for _, option := range opts {
		option(&op)
	}
	return &nacosResolver{cli: cli, opts: op}
}

接着查看Register

func (n *nacosRegistry) Register(info *registry.Info) error {
	if err := n.validateRegistryInfo(info); err != nil {
		return err
	}
	host, port, err := net.SplitHostPort(info.Addr.String())
	if err != nil {
		return fmt.Errorf("parse registry info addr error: %w", err)
	}
	p, err := strconv.Atoi(port)
	if err != nil {
		return fmt.Errorf("parse registry info port error: %w", err)
	}
	if host == "" || host == "::" {
		host, err = n.getLocalIpv4Host()
		if err != nil {
			return fmt.Errorf("parse registry info addr error: %w", err)
		}
	}

	_, e := n.cli.RegisterInstance(vo.RegisterInstanceParam{
		Ip:          host,
		Port:        uint64(p),
		ServiceName: info.ServiceName,
		Weight:      float64(info.Weight),
		Enable:      true,
		Healthy:     true,
		Metadata:    mergeTags(info.Tags, nacos.Tags),
		GroupName:   n.opts.group,
		ClusterName: n.opts.cluster,
		Ephemeral:   true,
	})
	if e != nil {
		return fmt.Errorf("register instance error: %w", e)
	}
	return nil
}

参数为注册数据

// Info is used for registry.
// The fields are just suggested, which is used depends on design.
type Info struct {
	// ServiceName will be set in kitex by default
	ServiceName string
	// Addr will be set in kitex by default
	Addr net.Addr
	// PayloadCodec will be set in kitex by default, like thrift, protobuf
	PayloadCodec string

	Weight    int
	StartTime time.Time
	WarmUp    time.Duration

	// extend other infos with Tags.
	Tags map[string]string
}

最终把数据读取到n.cli.RegisterInstance

_, e := n.cli.RegisterInstance(vo.RegisterInstanceParam{
		Ip:          host,
		Port:        uint64(p),
		ServiceName: info.ServiceName,
		Weight:      float64(info.Weight),
		Enable:      true,
		Healthy:     true,
		Metadata:    mergeTags(info.Tags, nacos.Tags),
		GroupName:   n.opts.group,
		ClusterName: n.opts.cluster,
		Ephemeral:   true,
	})

deRegister也是如此

func (n *nacosRegistry) Deregister(info *registry.Info) error {
	if err := n.validateRegistryInfo(info); err != nil {
		return err
	}
	host, port, err := net.SplitHostPort(info.Addr.String())
	if err != nil {
		return err
	}
	p, err := strconv.Atoi(port)
	if err != nil {
		return fmt.Errorf("parse registry info port error: %w", err)
	}
	if host == "" || host == "::" {
		host, err = n.getLocalIpv4Host()
		if err != nil {
			return fmt.Errorf("parse registry info addr error: %w", err)
		}
	}
	if _, err = n.cli.DeregisterInstance(vo.DeregisterInstanceParam{
		Ip:          host,
		Port:        uint64(p),
		ServiceName: info.ServiceName,
		Ephemeral:   true,
		GroupName:   n.opts.group,
		Cluster:     n.opts.cluster,
	}); err != nil {
		return err
	}
	return nil
}

综上，Kitex 框架保证了在核心链路中对服务信息的注册工作和注销动作，具体的注销逻辑要去到各自 registry-xxx 的包中实现。