gomicro v2--03 日志和中间件

一、如何自定义zap日志

Uber-go Zap(是非常快的、结构化的，分日志级别的Go日志库。

1.简单使用

（1）安装：

go get -u go.uber.org/zap

（2)使用：

2.自定义zap日志类

我经常需要把日志，打印到多个地方，如本地、控制台、远程日志系统如elk（一般通过kafka中转）；本地日志经常需要日志切开，我们可以把这些特性放到一个自定义日志类中。从而定义出属于自己系统的日志类。

（1）安装kafka

docker run -d --name zookeeper -p 2181:2181 wurstmeister/zookeeper

docker run -d --name kafka --publish 9092:9092 --link zookeeper --env KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 --env KAFKA_ADVERTISED_HOST_NAME=localhost --env KAFKA_ADVERTISED_PORT=9092 wurstmeister/kafka

(2) 编写日类

package zaplog

import (
	"github.com/Shopify/sarama"
	"github.com/natefinch/lumberjack"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"log"
	"os"
)

var zaplogger *zap.SugaredLogger

var (
	FileName       = "./latest.log"
	KafkaDsn       = "localhost:9092"
	FileMaxSize    = 10
	FileMaxBackups = 5
	FileMaxAge     = 30
	IsCompress     = false
)


func InitLogger() {
	encoder := getEncoder() //日志格式化方式
	//文件流
	fileWriteSyncer := getFileLogWriter()
	fileCore := zapcore.NewCore(encoder, fileWriteSyncer, zapcore.DebugLevel)
	//控制台流
	consoleWriteSyncer := getConsoleLogWriter()
	consoleCore := zapcore.NewCore(encoder, consoleWriteSyncer, zapcore.DebugLevel)
	卡夫卡流
	kafkaWriteSyncer := getKafkaLogWriter()
	kafkaCore := zapcore.NewCore(encoder, kafkaWriteSyncer, zapcore.DebugLevel)

	logger := zap.New(zapcore.NewTee(fileCore, consoleCore, kafkaCore))
	zaplogger = logger.Sugar()
}

func getEncoder() zapcore.Encoder {
	encoderConfig := zap.NewProductionEncoderConfig()
	encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
	encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder
	return zapcore.NewConsoleEncoder(encoderConfig)
}

func getFileLogWriter() zapcore.WriteSyncer {
	lumberJackLogger := &lumberjack.Logger{
		Filename:   FileName,
		MaxSize:    FileMaxSize,
		MaxBackups: FileMaxBackups,
		MaxAge:     FileMaxAge,
		Compress:   IsCompress,
	}
	return zapcore.AddSync(lumberJackLogger)
}

func getConsoleLogWriter() zapcore.WriteSyncer {
	return zapcore.AddSync(os.Stdout)
}

type KafkaWriter struct {
	topic        string
	syncProducer sarama.SyncProducer
}

func (kp *KafkaWriter) Write(p []byte) (n int, err error) {
	_, _, err = kp.syncProducer.SendMessage(&sarama.ProducerMessage{
		Topic: kp.topic,
		Value: sarama.ByteEncoder(p),
	})
	if err != nil {
		log.Println(err)
		return 0, err
	}
	//fmt.Println("send msgs:",string(p))
	return len(p), nil

}

func getKafkaLogWriter() zapcore.WriteSyncer {
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForLocal
	config.Producer.Return.Successes = true
	syncProducer, err := sarama.NewSyncProducer([]string{KafkaDsn}, config)
	if err != nil {
		log.Fatal(err)
	}
	return zapcore.AddSync(&KafkaWriter{topic: "zaplog", syncProducer: syncProducer})
}

//对外接口,这几个函数只是为了方便使用，也可以用一个名字很短的函数直接返回 zaplogger

func Info(args ...interface{}) {
	zaplogger.Info(args)
}

func Error(args ...interface{}) {
	zaplogger.Error(args)
}

func Debug(args ...interface{}) {
	zaplogger.Debug(args)
}

func Warn(args ...interface{}) {
	zaplogger.Warn(args)
}

func Fatal(args ...interface{}) {
	zaplogger.Fatal(args)
}

func With(args ...interface{}) *zap.SugaredLogger {
	return zaplogger.With(args)
}

（3）编写单元测试

package zaplog

import (
	"testing"
)

func TestWriter(t *testing.T) {
	InitLogger()
	Info("hhhh","日志打出来了")
	Info("wwww","日志打出来了喔")
}

 二、如何扩展go micro的扩展

1.plugin

Go Micro 构建于 Go 接口之上. 因此这些接口的实现是可插拔的.

默认情况下, go-micro 只提供核心上每个接口的几个实现, 但它是完全可插拔的. 已经有几十个插件, 在这里 github.com/micro/go-plugins 欢迎贡献! 插件可确保 Go Micro 服务在技术发展后长期保持生存。如果要集成插件, 只需将它们链接到单独的文件中并重新生成

创建 plugins.go 文件并导入所需的插件:

package main

import (
        // consul registry
        _ "github.com/micro/go-plugins/registry/consul"
        // rabbitmq transport
        _ "github.com/micro/go-plugins/transport/rabbitmq"
        // kafka broker
        _ "github.com/micro/go-plugins/broker/kafka"
)

通过包含插件文件来构建应用程序:

# assuming files main.go and plugins.go are in the top level dir

# For local use
go build -o service *.go

插件的标志用法:

service --registry=etcdv3 --transport=nats --broker=kafka

或者首选的是使用环境变量来处理 12 因素的应用程序

或者您可以将插件设置为直接在代码中服务的选项

package main

import (
        "github.com/micro/go-micro/v2" 
        // consul registry
        "github.com/micro/go-plugins/registry/consul"
        // rabbitmq transport
        "github.com/micro/go-plugins/transport/rabbitmq"
        // kafka broker
        "github.com/micro/go-plugins/broker/kafka"
)

func main() {
    registry := consul.NewRegistry()
    broker := kafka.NewBroker()
    transport := rabbitmq.NewTransport()

        service := micro.NewService(
                micro.Name("greeter"),
                micro.Registry(registry),
                micro.Broker(broker),
                micro.Transport(transport),
        )

    service.Init()
    service.Run()
}

插件是一个建立在 Go 接口上的概念. 每个包都维护一个高级接口抽象. 只需实现接口并将其作为服务选项传递给它即可.

服务发现接口称为 Registry. 实现此接口的任何内容都可以用作注册表. 这同样适用于其他包.

type Registry interface {
    Register(*Service, ...RegisterOption) error
    Deregister(*Service) error
    GetService(string) ([]*Service, error)
    ListServices() ([]*Service, error)
    Watch() (Watcher, error)
    String() string
}

参阅 go-plugins 更好地了解实现细节.

2.wrapper

相当于，go web中的中间件，可以再客户端发起请求前、或者服务端处理请求前，做出一些公共的处理编写方式如下：

（1）客户端：

引入：

CustomerServer = micro.NewService(
		micro.Name(serviceName),
		micro.Version("latest"),
		micro.Registry(etcd.NewRegistry(
			registry.Addrs(addr),
		)),
	 
		micro.WrapClient(
			 ....//此处引入，你的客户端代码 如 ：                
             microOpentracing.NewClientWrapper(opentracing.GlobalTracer())
		),
	)

编写：

// NewClientWrapper accepts an open tracing Trace and returns a Client Wrapper
func NewClientWrapper(ot opentracing.Tracer) client.Wrapper {
	return func(c client.Client) client.Client {
		if ot == nil {
			ot = opentracing.GlobalTracer()
		}
		return &otWrapper{ot, c}
	}
}

（2）服务端：

引入：

 
	// New Server
	CustomerServer = micro.NewService(
		micro.Name(serviceName),
		micro.Version("latest"),
		micro.Registry(etcd.NewRegistry(
			registry.Addrs(addr),
		)),
		micro.WrapHandler(
			microOpentracing.NewHandlerWrapper(opentracing.GlobalTracer()), //此处引入服务端wrapper
		),
	)
 

编写：

// NewHandlerWrapper accepts an opentracing Tracer and returns a Handler Wrapper
func NewHandlerWrapper(ot opentracing.Tracer) server.HandlerWrapper {
	return func(h server.HandlerFunc) server.HandlerFunc {
		return func(ctx context.Context, req server.Request, rsp interface{}) error {
			if ot == nil {
				ot = opentracing.GlobalTracer()
			}
			name := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint())
			ctx, span, err := StartSpanFromContext(ctx, ot, name)
			if err != nil {
				return err
			}
			defer span.Finish()
			if err = h(ctx, req, rsp); err != nil {
				span.LogFields(opentracinglog.String("error", err.Error()))
				span.SetTag("error", true)
			}
			return err
		}
	}
}

3.参考示例

go micro的很多功能，都可以再下面的项目中，找到参考示例

go micro 实例代码 ：GitHub - microhq/examples: Go Micro examples. Moved to go-micro/examples.

三、公共日志处理

对于1.web请求的请求参数，返回值；2.grpc(go micro)公共请求参数，返回值。我们可以通过中间件的方式，再一个地方记录，而不需要每个接口和调用都处理。还有错误日志等等。

下边我们将介绍同过 gin中间件和 go micro定义这些公共日志处理。  

1. gin access日志记录中间件

 zaplog/zaplog_gin_middleware.go

package zaplog

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/gin-gonic/gin"
	"time"
)

type bodyLogWriter struct {
	gin.ResponseWriter
	body *bytes.Buffer
}

func (w bodyLogWriter) Write(b []byte) (int, error) {
	w.body.Write(b)
	return w.ResponseWriter.Write(b)
}
func (w bodyLogWriter) WriteString(s string) (int, error) {
	w.body.WriteString(s)
	return w.ResponseWriter.WriteString(s)
}

type AccessLog struct {
	RequestTime     interface{}
	RequestMethod   interface{}
	RequestUrl      interface{}
	RequestProto    interface{}
	RequestUa       interface{}
	RequestReferer  interface{}
	RequestPostData interface{}
	RequestPostBody interface{}
	RequestClientIp interface{}
	ResponseTime    interface{}
	ResponseBody    interface{}
	CostTime        interface{}
}

func ZaplogGinMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {

		bodyLogWriter := &bodyLogWriter{body: bytes.NewBufferString(""), ResponseWriter: c.Writer}
		c.Writer = bodyLogWriter //拦截write使可重复读取返回值
		//开始时间 ms
		startTime := time.Now().UnixNano() / 1e6
		//处理请求
		c.Next()
		responseBody := bodyLogWriter.body.String()
		//结束时间
		endTime := time.Now().UnixNano() / 1e6
		if c.Request.Method == "POST" {
			c.Request.ParseForm()
		}
		//日志格式
		var accessLog AccessLog
		accessLog.RequestTime = startTime
		accessLog.RequestMethod = c.Request.Method
		accessLog.RequestUrl = c.Request.RequestURI
		accessLog.RequestProto = c.Request.Proto
		accessLog.RequestUa = c.Request.UserAgent()
		accessLog.RequestReferer = c.Request.Referer()
		accessLog.RequestPostData = c.Request.PostForm.Encode()
		body, _ := c.GetRawData()
		accessLog.RequestPostBody = string(body)
		accessLog.RequestClientIp = c.ClientIP()
		accessLog.ResponseTime = endTime
		accessLog.ResponseBody = responseBody
		accessLog.CostTime = fmt.Sprintf("%vms", endTime-startTime)
		str, _ := json.Marshal(accessLog)

		Info("access_log  ", string(str))
	}
}

2.go micro 服务访问日志中间件

 zaplog/zaplog_gomicro_wrapper.go

package zaplog

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/micro/go-micro/v2/server"
	"time"
)

func ZaplogMicroWrapper(fn server.HandlerFunc) server.HandlerFunc {
	return func(ctx context.Context, req server.Request, rsp interface{}) error {
		//开始时间 ms
		startTime := time.Now().UnixNano() / 1e6
		err := fn(ctx, req, rsp)
		//结束时间
		endTime := time.Now().UnixNano() / 1e6
		//日志格式
		accessLogMap := make(map[string]interface{})
		accessLogMap["request_time"] = startTime
		accessLogMap["request_body"] = req.Body()
		accessLogMap["request_service"] = req.Service()
		accessLogMap["request_method"] = req.Method()
		accessLogMap["response_time"] = endTime
		accessLogMap["response"] = rsp
		accessLogMap["cost_time"] = fmt.Sprintf("%vms", endTime-startTime)
		str, _ := json.Marshal(accessLogMap)

		Info("micro_access ", string(str))
		return err
	}
}

3.使用

（1）gateway中使用

main.go

package main

import (
	"gitee.com/gudongkun/haveclient_customer/client/customer_server"
	"gitee.com/gudongkun/haveclient_customer/client/users"
	"gitee.com/gudongkun/haveclient_gateway/zaplog"
	"github.com/gin-gonic/gin"
	"io/ioutil"
)
func main () {
	
	customer_server.InitService("have.client.service.customer","127.0.0.1:2379")
	zaplog.InitLogger() //初始化zap日志

    //关闭gin 日志
	 //gin.SetMode(gin.ReleaseMode)
	gin.DefaultWriter = ioutil.Discard

	r := gin.Default()
	r.Use(zaplog.ZaplogGinMiddleware()) //记录accesslog中间件。
	r.GET("/", func(c *gin.Context){
		u,_ := users.GetUser(c.Request.Context(),1)
		c.JSON(200,u)
	})
	r.Run(":8088")   // 强指定端口，默认8088
}

(2) customer Service使用

package customer_server

import (
	"gitee.com/gudongkun/haveclient_customer/zaplog"
	"github.com/micro/go-micro/v2"
	"github.com/micro/go-micro/v2/registry"
	"github.com/micro/go-micro/v2/registry/etcd"
	microOpentracing "github.com/micro/go-plugins/wrapper/trace/opentracing/v2"
	"github.com/opentracing/opentracing-go"
)

var CustomerServer micro.Service
var ServerName string


//InitClientService 初始化函数
//参数：
// 	serviceName   string 服务名
//  addr string   etcd 地址
func InitService(serviceName, addr string) {
	ServerName = serviceName
	// New Server
	CustomerServer = micro.NewService(
		micro.Name(serviceName),
		micro.Version("latest"),
		micro.Registry(etcd.NewRegistry(
			registry.Addrs(addr),
		)),
		micro.WrapHandler(
			microOpentracing.NewHandlerWrapper(opentracing.GlobalTracer()), //服务端开启 span
			zaplog.ZaplogMicroWrapper,
		),
		micro.WrapClient(
			// 配置链路追踪为jaeger
			microOpentracing.NewClientWrapper(opentracing.GlobalTracer()), //客户端开启 span ，一般服务端或客户端开启一个就可以
		),
	)
	//CustomerServer.Init()
}