golang http服务实现多ip监听，及优雅重启

在工作中，有时需要对http服务实现多监听，http服务重启等需求。大多数web框架只实现的是单ip监听，要实现多ip监听就需要循环监听ip；

而重启http服务，首先想到的是用endless来优雅的实现服务的重启，但是当多ip监听时，一个项目不能用一个endLess,多了会报错，且windows环境也无法实现重启；

所以，我在工作中最终使用了gracehttp(grace)来实现多ip监听及优雅的重启，但是grace也是只能做到linux的重启和启动，应为window没有定义signal，这里需要吐槽一下window；最借鉴了一下另外一个开源包window gracehttp，是多grace的扩展，实现了http服务在window的多ip监听，但是重启一样报错(window 不支持signal), 只能再次修改开源包.

话不多说上代码：

linux版：

//go:build !windows
// +build !windows

package utils

import (
	"crypto/tls"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/labstack/echo/v4"
	"net/http"
	"sync"
	"time"
)

/**
http server 优雅地重启
*/

var grace *GraceHttp
var gOnce sync.Once
var SysRestart = make(chan int)

type GraceHttp struct {
	SrvList sync.Map
}

func NewGrace() *GraceHttp {
	gOnce.Do(func() {
		grace = &GraceHttp{}
	})
	return grace
}

func (g *GraceHttp) AddService(name string, srv *http.Server) {
	g.SrvList.Store(name, srv)
}

func (g *GraceHttp) Run() {
	srvs := make([]*http.Server, 0)
	g.SrvList.Range(func(key, value any) bool {
		srvs = append(srvs, value.(*http.Server))
		g.SrvList.Delete(key)
		return true
	})

    // 调用grace启动
	if err := gracehttp.ServeWithOptions(srvs); err != nil {
		panic(err)
	}
}

func InitServer(address string, router *echo.Echo, cfg *tls.Config) *http.Server {
	s := &http.Server{
		Addr:           address,
		Handler:        router,
		ReadTimeout:    20 * time.Second,
		WriteTimeout:   20 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	if cfg != nil {
		s.TLSConfig = cfg
	}
	return s
}

windows版:

//go:build windows
// +build windows

package utils

import (
	"crypto/tls"
	"github.com/labstack/echo/v4"
	"net/http"
	"sync"
	"time"
)

var grace *GraceHttp
var gOnce sync.Once
var SysRestart = make(chan int)

type GraceHttp struct {
	SrvList sync.Map
}

func NewGrace() *GraceHttp {
	gOnce.Do(func() {
		grace = &GraceHttp{}
	})
	return grace
}

func (g *GraceHttp) AddService(name string, srv *http.Server) {
	g.SrvList.Store(name, srv)
}

func (g *GraceHttp) Run() {
	srvs := make([]*http.Server, 0)
	g.SrvList.Range(func(key, value any) bool {
		srvs = append(srvs, value.(*http.Server))
		g.SrvList.Delete(key)
		return true
	})
	// g.SrvList = sync.Map{}
    // 使用改版的grace
	if err := Serve(srvs...); err != nil {
		panic(err)
	}
}

func InitServer(address string, router *echo.Echo, cfg *tls.Config) *http.Server {
	s := &http.Server{
		Addr:           address,
		Handler:        router,
		ReadTimeout:    20 * time.Second,
		WriteTimeout:   20 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	if cfg != nil {
		s.TLSConfig = cfg
	}
	return s
}

改版的windows grace,注这是对开源代码的改版。

package utils

import (
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"github.com/facebookgo/grace/gracenet"
	"github.com/facebookgo/httpdown"
	"log"
	"net"
	"net/http"
	"os"
	"sync"
	"syscall"
)

var (
	verbose    = flag.Bool("gracehttp.log", true, "Enable logging.")
	didInherit = os.Getenv("LISTEN_FDS") != ""
	ppid       = os.Getppid()
)

const SIGUSR2 = syscall.Signal(0x1f)

// An app contains one or more servers and associated configuration.
type app struct {
	servers   []*http.Server
	http      *httpdown.HTTP
	net       *gracenet.Net
	listeners []net.Listener
	sds       []httpdown.Server
	errors    chan error
}

func newApp(servers []*http.Server) *app {
	return &app{
		servers:   servers,
		http:      &httpdown.HTTP{},
		net:       &gracenet.Net{},
		listeners: make([]net.Listener, 0, len(servers)),
		sds:       make([]httpdown.Server, 0, len(servers)),

		// 2x num servers for possible Close or Stop errors + 1 for possible
		// StartProcess error.
		errors: make(chan error, 1+(len(servers)*2)),
	}
}

func (a *app) listen() error {
	for _, s := range a.servers {
		// TODO: default addresses
		l, err := a.net.Listen("tcp", s.Addr)
		if err != nil {
			return err
		}
		if s.TLSConfig != nil {
			l = tls.NewListener(l, s.TLSConfig)
		}
		a.listeners = append(a.listeners, l)
	}
	return nil
}

func (a *app) serve() {
	for i, s := range a.servers {
		a.sds = append(a.sds, a.http.Serve(s, a.listeners[i]))
	}
}

func (a *app) wait() {
	var wg sync.WaitGroup
	fmt.Println("wg lent:", len(a.sds))
	wg.Add(len(a.sds) * 2) // Wait & Stop
	fmt.Printf("wg:%+v", wg)
	go a.signalHandler(&wg)
	for _, s := range a.sds {
		go func(s httpdown.Server) {
			fmt.Println("等待启动.........")
			defer wg.Done()
			if err := s.Wait(); err != nil {
				a.errors <- err
			}
		}(s)
	}
	wg.Wait()
}

var WinRestart = make(chan int)

//func (a *app) term(wg *sync.WaitGroup) {
//	for _, s := range a.sds {
//		go func(s httpdown.Server) {
//			defer wg.Done()
//			if err := s.Stop(); err != nil {
//				a.errors <- err
//			}
//		}(s)
//	}
//}

func (a *app) signalHandler(wg *sync.WaitGroup) {
	select {
	case <-WinRestart:
		fmt.Println("收到重启请求....")
		for _, s := range a.sds {
			fmt.Println("关闭http。。。。。。。。")
			if err := s.Stop(); err != nil {
				a.errors <- err
			}
			wg.Done()
		}
	}
	SysRestart <- 1
}

// Serve will serve the given http.Servers and will monitor for signals
// allowing for graceful termination (SIGTERM) or restart (SIGUSR2).
func Serve(servers ...*http.Server) error {
	a := newApp(servers)

	// Acquire Listeners
	if err := a.listen(); err != nil {
		return err
	}

	// Some useful logging.
	if *verbose {
		if didInherit {
			if ppid == 1 {
				log.Printf("Listening on init activated %s", pprintAddr(a.listeners))
			} else {
				const msg = "Graceful handoff of %s with new pid %d and old pid %d"
				log.Printf(msg, pprintAddr(a.listeners), os.Getpid(), ppid)
			}
		} else {
			const msg = "Serving %s with pid %d"
			log.Printf(msg, pprintAddr(a.listeners), os.Getpid())
		}
	}

	// Start serving.
	a.serve()

	// Close the parent if we inherited and it wasn't init that started us.
	if didInherit && ppid != 1 {
		if err := terminateProcess(ppid); err != nil {
			return fmt.Errorf("failed to close parent: %s", err)
		}
	}

	waitdone := make(chan struct{})
	go func() {
		defer close(waitdone)
		a.wait()
	}()

	select {
	case err := <-a.errors:
		if err == nil {
			panic("unexpected nil error")
		}
		return err
	case <-waitdone:
		if *verbose {
			log.Printf("Exiting pid %d.", os.Getpid())
		}
		return nil
	}
}

// Used for pretty printing addresses.
func pprintAddr(listeners []net.Listener) []byte {
	var out bytes.Buffer
	for i, l := range listeners {
		if i != 0 {
			fmt.Fprint(&out, ", ")
		}
		fmt.Fprint(&out, l.Addr())
	}
	return out.Bytes()
}

func terminateProcess(pid int) error {
	process, err := os.FindProcess(pid)
	if err != nil {
		return err
	}

	return process.Signal(syscall.SIGTERM)
}

在需要重启的地方调用重启方法：

func Reload() error {
	linuxVersion, err := GetLinuxPlatformFamily()
	if err != nil {
		return err
	}
	var cmd *exec.Cmd
	pid := os.Getpid()
	switch linuxVersion {
	case "windows":
		go func() {
			fmt.Println("发送重启命令")
			WinRestart <- 1
		}()
		return nil
	case LINUXRHEL:
		cmd = exec.Command("bash", "-c", "rpm -qa | grep -i supervisor")
	case LINUXDEBIAN:
		cmd = exec.Command("bash", "-c", "service --status-all |grep supervisor")
	}
	var out bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &stderr
	cmd.Run()
	// 安装了supervisor,就让supervisor拉起，否则重启
	if out.String() != "" && strings.Contains(out.String(), "[ + ]  supervisor") {
		os.Exit(0)
		return nil
	}

	cmd = exec.Command("kill", "-USR2", strconv.Itoa(pid))
	return cmd.Run()
}

最后在main里面实现一个守护进程，进行windows的重启：

// 守护进程用于windows重启
func (e *Engine) windowsDeamon() {
	if runtime.GOOS == "windows" {
		for {
			select {
			case <-utils.SysRestart:
				fmt.Println("windows deamon program server restart................")
				x.SafeGo(func() {
					e.runBackend()
					e.api.Start()
					utils.NewGrace().Run()
				})
			}
		}
	}
}

最后就实现了，http 服务在windows和linux的多ip监听及优雅重启。 

注意事项：

在wg.done后面执行channel之类的阻塞，可能会出现wg.done成负值错误，需要控制在毫米级以下。