Kubernetes Nginx Ingress Controller源码分析

main

controllers/nginx/pkg/cmd/controller/main.go:29

func main() {
    // start a new nginx controller
    ngx := newNGINXController()
    // create a custom Ingress controller using NGINX as backend
    ic := controller.NewIngressController(ngx)
    go handleSigterm(ic)
    // start the controller
    ic.Start()
    // wait
    glog.Infof("shutting down Ingress controller...")
    for {
        glog.Infof("Handled quit, awaiting pod deletion")
        time.Sleep(30 * time.Second)
    }
}

• start a new nginx controller.
• create a custom Ingress controller using NGINX as backend.
• start the Ingress controller.

newNGINXController

controllers/nginx/pkg/cmd/controller/nginx.go:68

func newNGINXController() ingress.Controller {

    // 从环境变量“NGINX_BINARY”中获取nginx二进制文件的路径，如果没有该环境变量，则使用预设的默认值“/usr/sbin/nginx”
    ngx := os.Getenv("NGINX_BINARY")
    if ngx == "" {
        ngx = binary
    }

    // 从"/etc/resolv.conf"中读取dns nameservers的IP列表
    h, err := dns.GetSystemNameServers()
    ...

    // 构造NGINXController对象
    n := &NGINXController{
        binary:        ngx,
        configmap:     &api_v1.ConfigMap{},

        // 查看 "/proc/net/if_inet6"文件是否存在，检查是否Enable IPv6
        isIPV6Enabled: isIPv6Enabled(),

        resolver:      h,
        proxy: &proxy{

          // 设置proxy的default server为 ”127.0.0.1:442”
            Default: &server{
                Hostname:      "localhost",
                IP:            "127.0.0.1",
                Port:          442,
                ProxyProtocol: true,
            },
        },
    }

    // 启动对 tcp/443 端口的监听
    listener, err := net.Listen("tcp", ":443")

    proxyList := &proxyproto.Listener{Listener: listener}

    // start goroutine that accepts tcp connections in port 443
    go func() {
        for {
            var conn net.Conn
            var err error

            if n.isProxyProtocolEnabled {
                // we need to wrap the listener in order to decode
                // proxy protocol before handling the connection
                conn, err = proxyList.Accept()
            } else {
                conn, err = listener.Accept()
            }

            if err != nil {
                glog.Warningf("unexpected error accepting tcp connection: %v", err)
                continue
            }

            glog.V(3).Infof("remote address %s to local %s", conn.RemoteAddr(), conn.LocalAddr())
            go n.proxy.Handle(conn)
        }
    }()

    // onChange定义nginx.tmpl文件内容发生变化时需要进行操作：重新生成nginx template instance,并赋给NGINXController
    var onChange func()
    onChange = func() {
        template, err := ngx_template.NewTemplate(tmplPath, onChange)
        ...

        n.t.Close()
        n.t = template
        glog.Info("new NGINX template loaded")
    }

    // 根据nginx.tmpl生成nginx Template，并将onChange注册进去，通过FileWatcher监听nginx.tmpl的变化时，自动调用onChange。
    ngxTpl, err := ngx_template.NewTemplate(tmplPath, onChange)

    n.t = ngxTpl

    // start a new NGINX master process running in foreground.
    go n.Start()

    return ingress.Controller(n)
}

• 从环境变量“NGINX_BINARY”中获取nginx二进制文件的路径，如果没有该环境变量，则使用预设的默认值“/usr/sbin/nginx”

• 从”/etc/resolv.conf”中读取dns nameservers的IP列表

• 构造NGINXController对象

  • 查看 “/proc/net/if_inet6”文件是否存在，检查是否Enable IPv6
  • 设置proxy的default server为 ”127.0.0.1:442”

• 启动对 tcp/443 端口的监听

• start goroutine that accepts tcp connections in port 443

• onChange定义nginx.tmpl文件内容发生变化时需要进行操作：重新生成nginx template instance,并赋给NGINXController

• 根据nginx.tmpl生成nginx Template，并将onChange注册进去，通过FileWatcher监听nginx.tmpl的变化时，自动调用onChange。

• start a new NGINX master process running in foreground.

NGINXController的定义如下：

controllers/nginx/pkg/cmd/controller/nginx.go:156

type NGINXController struct {
    t *ngx_template.Template

    configmap *api_v1.ConfigMap

    storeLister ingress.StoreLister

    binary   string
    resolver []net.IP

    cmdArgs []string

    stats        *statsCollector
    statusModule statusModule

    // returns true if IPV6 is enabled in the pod
    isIPV6Enabled bool

    // returns true if proxy protocol es enabled
    isProxyProtocolEnabled bool

    proxy *proxy
}

NewTemplate时，我们可以看到根据nginx.tmpl来生成nginx配置时，注册了funcMap，从funcMap我们可以看出nginx.tpml主要包括哪些组成部分。

//controllers/nginx/pkg/template/template.go:57
func NewTemplate(file string, onChange func()) (*Template, error) {
    tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file)

    fw, err := watch.NewFileWatcher(file, onChange)

    return &Template{
        tmpl:      tmpl,
        fw:        fw,
        s:         defBufferSize,
        tmplBuf:   bytes.NewBuffer(make([]byte, 0, defBufferSize)),
        outCmdBuf: bytes.NewBuffer(make([]byte, 0, defBufferSize)),
    }, nil
}

//controllers/nginx/pkg/template/template.go:123
funcMap = text_template.FuncMap{
    "empty": func(input interface{}) bool {
        check, ok := input.(string)
        if ok {
            return len(check) == 0
        }
        return true
    },
    "buildLocation":            buildLocation,
    "buildAuthLocation":        buildAuthLocation,
    "buildAuthResponseHeaders": buildAuthResponseHeaders,
    "buildProxyPass":           buildProxyPass,
    "buildRateLimitZones":      buildRateLimitZones,
    "buildRateLimit":           buildRateLimit,
    "buildResolvers":           buildResolvers,
    "buildUpstreamName":        buildUpstreamName,
    "isLocationAllowed":        isLocationAllowed,
    "buildLogFormatUpstream":   buildLogFormatUpstream,
    "buildDenyVariable":        buildDenyVariable,
    "getenv":                   os.Getenv,
    "contains":                 strings.Contains,
    "hasPrefix":                strings.HasPrefix,
    "hasSuffix":                strings.HasSuffix,
    "toUpper":                  strings.ToUpper,
    "toLower":                  strings.ToLower,
    "formatIP":                 formatIP,
    "buildNextUpstream":        buildNextUpstream,
}

controller.NewIngressController

core/pkg/ingress/controller/launch.go:29

// NewIngressController returns a configured Ingress controller
func NewIngressController(backend ingress.Controller) *GenericController {

    ... 

    flags.Parse(os.Args)

    // 对于Ningx而言，OverrideFlags主要处理ingress-class配置，如果ingress-class没有配置，则使用默认配置"nginx"，并启动Prometheus Collector for the nginx。
    backend.OverrideFlags(flags)

    flag.Set("logtostderr"