k8s ingress详解（2）

这一篇看一下ingress controller的实现

ingress controller是一个守护进程，部署为一个Kubernetes Pod

ingress controller通过watch ingresses接口，动态更新ingress的资源

官方的ingress controller实现内容：
1.从APIserver拉去配置信息
2.基于golang text/template 模块书写Nginx配置模板
3.重新加载nginx

官方Nginx ingress controller 定义

const (
    nginxConf = `
		events {
		  worker_connections 1024;
		}
		http {
			{{range $ing := .Items}}
				{{range $rule := $ing.Spec.Rules}}
				  server {
				    listen 80;
				    server_name {{$rule.Host}};
					{{ range $path := $rule.HTTP.Paths }}
					    location {{$path.Path}} {
					      proxy_set_header Host $host;
					      proxy_pass http://{{$path.Backend.ServiceName}}.{{$ing.Namespace}}.svc.cluster.local:{{$path.Backend.ServicePort}};
					    }
					{{end}}
				  }
				{{end}}
			{{end}}
		}`
)

这里通过定义nginxConf常量，实际内容为Nginx的配置模板

#配置加载

func (ngx *Manager) CheckAndReload(cfg config.Configuration, ingressCfg ingress.Configuration) error {
	ngx.reloadRateLimiter.Accept()
	ngx.reloadLock.Lock()
	defer ngx.reloadLock.Unlock()
	newCfg, err := ngx.template.Write(cfg, ingressCfg, ngx.testTemplate)
	if err != nil {
		return fmt.Errorf("failed to write new nginx configuration. Avoiding reload: %v", err)
	}
	changed, err := ngx.needsReload(newCfg)
	if err != nil {
		return err
	}
	if changed {
		if err := ngx.shellOut("nginx -s reload"); err != nil {
			return fmt.Errorf("error reloading nginx: %v", err)
		}
		glog.Info("change in configuration detected. Reloading...")
	}
	return nil
}

CheckAndReload的第一个参数的Nginx的全局配置参数，对于一个ingress controller启动后一般这些参数是不变的，我们关注的ingress的变化
ngx.template.Write(cfg, ingressCfg, ngx.testTemplate)实现了将ingress渲染到Nginx配置文件的过程
ngx.testTemplate用于测试配置文件是否正确，实际就是Nginx -t
ngx.template.Write就是调用的k8s.io\contrib\ingress\controllers\nginx\nginx\template\template.go Template.Write(cfg  config.Configuration,ingressCfg ingress.Configuration,isValidTemplate func([]byte) error) 函数

最终的配置conf 类型为 make(map[string]interface{})，一般的nginx用到的结构有：

type Configuration struct {
	Upstreams>     []*Upstream
	Servers      []*Server
	TCPUpstreams []*Location
	UDPUpstreams []*Location
}
type Upstream struct {
	Name     string
	Backends []UpstreamServer
	Secure   bool
}
type UpstreamServer struct {
	Address     string
	Port        string
	MaxFails    int
	FailTimeout int
}
type Server struct {
	Name              string
	Locations         []*Location
	SSL               bool
	SSLCertificate    string
	SSLCertificateKey string
	SSLPemChecksum    string
}
type Location struct {
	Path            string
	IsDefBackend    bool
	Upstream        Upstream
	Auth            auth.Nginx
	RateLimit       ratelimit.RateLimit
	Redirect        rewrite.Redirect
	SecureUpstream  bool
	Whitelist       ipwhitelist.SourceRange
	EnableCORS      bool
	ExternalAuthURL authreq.Auth
}

那么ingress.Configuration是怎么产生的呢

func (lbc *loadBalancerController) sync(key string) error {
	if !lbc.controllersInSync() {
		time.Sleep(podStoreSyncedPollPeriod)
		return fmt.Errorf("deferring sync till endpoints controller has synced")
	}

	// by default no custom configuration configmap
	cfg := &api.ConfigMap{}

	if lbc.nxgConfigMap != "" {
		// Search for custom configmap (defined in main args)
		var err error
		ns, name, _ := parseNsName(lbc.nxgConfigMap)
		cfg, err = lbc.getConfigMap(ns, name)
		if err != nil {
			return fmt.Errorf("unexpected error searching configmap %v: %v", lbc.nxgConfigMap, err)
		}
	}

	ngxConfig := lbc.nginx.ReadConfig(cfg)
	ngxConfig.HealthzURL = lbc.defHealthzURL

	ings := lbc.ingLister.Store.List()
	upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings)

	return lbc.nginx.CheckAndReload(ngxConfig, ingress.Configuration{
		Upstreams:    upstreams,
		Servers:      servers,
		TCPUpstreams: lbc.getTCPServices(),
		UDPUpstreams: lbc.getUDPServices(),
	})
}

lbc.ingLister.Store.List()获取到了最新ingress配置

那这个函数又是怎么被调用到的

lbc.ingLister.Store, lbc.ingController = framework.NewInformer(
	&cache.ListWatch{
		ListFunc:  ingressListFunc(lbc.client, namespace),
		WatchFunc: ingressWatchFunc(lbc.client, namespace),
	},
	&extensions.Ingress{}, resyncPeriod, ingEventHandler)

通过listwatch机制检测ingress资源
当有增删改等动作时都会调用lbc.syncQueue.enqueue(obj)

lbc.syncQueue实际上是通过NewTaskQueue 函数转变loadBalancerController.sync而来，每次lbc.syncQueue被调用时loadBalancerController.sync都会被调用

func (lbc *loadBalancerController) Run() {
	glog.Infof("starting NGINX loadbalancer controller")
	go lbc.nginx.Start()

	go lbc.ingController.Run(lbc.stopCh)
	go lbc.endpController.Run(lbc.stopCh)
	go lbc.svcController.Run(lbc.stopCh)
	go lbc.secrController.Run(lbc.stopCh)
	go lbc.mapController.Run(lbc.stopCh)

	go lbc.syncQueue.run(time.Second, lbc.stopCh)
	go lbc.ingQueue.run(time.Second, lbc.stopCh)

	<-lbc.stopCh
}

go lbc.syncQueue.run(time.Second, lbc.stopCh)实际运行函数为worker，调用了sync

func (t *taskQueue) worker() {
	for {
		key, quit := t.queue.Get()
		if quit {
			close(t.workerDone)
			return
		}
		glog.V(3).Infof("syncing %v", key)
		if err := t.sync(key.(string)); err != nil {
			glog.Warningf("requeuing %v, err %v", key, err)
			t.requeue(key.(string))
		} else {
			t.queue.Forget(key)
		}

		t.queue.Done(key)
	}
}

其他关于secret,configmap,service,endpoint的controller不再赘述

按照官方的nginx ingress controller实现我们可以发现，实际上就是通过api与apiserver与api进行交互，监控configmap secret service endpoint等信息的变化进行加载然后reload的过程。