云原生边缘计算解决方案之openyurt

架构解析(openyurt)

组件

• master组件：
  • yurt-controller-manager
  • yurt-tunnel-server
  • yurt app manager
• 边缘worker组件
  • yurt-tunnel-agent
  • yurt-hub

 

tunnel原理解析

原生k8s：client(kubectl exec,log等) => kube-apiserver:6443 => kubelet-node:10250

边缘k8s： client(kubectl exec,log等) => kube-apiserver:6443 => yurt-tunnel-server:10263=> yurt-tunnel-agent => kubelet-node:10250

openyurt解决思路：通过iptables截取流量的方式解决 如下是iptables的链路转发的整个流程：

Chain OUTPUT:
     TUNNEL-PORT  tcp  --  0.0.0.0/0            0.0.0.0/0            /* yurttunnel server port */
     
Chain TUNNEL-PORT:
     TUNNEL-PORT-10255  tcp  --  0.0.0.0/0            0.0.0.0/0            tcp dpt:10255 /* jump to port 10255 */
     TUNNEL-PORT-10250  tcp  --  0.0.0.0/0            0.0.0.0/0            tcp dpt:10250 /* jump to port 10250 */
     
Chain TUNNEL-PORT-10255 
     RETURN     tcp  --  0.0.0.0/0            127.0.0.1            /* return request to access node directly */ tcp dpt:10250
     RETURN     tcp  --  0.0.0.0/0            172.16.1.199         /* return request to access node directly */ tcp dpt:10250
     DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0            /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263

Chain TUNNEL-PORT-10250
     RETURN     tcp  --  0.0.0.0/0            127.0.0.1            /* return request to access node directly */ tcp dpt:10250
     RETURN     tcp  --  0.0.0.0/0            172.16.1.199         /* return request to access node directly */ tcp dpt:10250
     DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0            /* dnat to tunnel for access node */ tcp dpt:10250 to:172.16.1.199:10263

 

yurt-hub原理解析

作用：用作来自Kubernetes节点守护程序（Kubelet，Kubeproxy，CNI插件等）的出站流量的代理，并配有本地缓存机制，当节点无法通信apiserver的时候 不影响该节点正常提供业务达到边缘自治的功能并且yurt-hub守护程序可以帮助节点在重新启动后恢复状态。

目前版本缺陷 无法代理hostnetwork=true的pod

原因：为了保证安全性 yurthub只透出了127.0.0.1:10261端口。

解决：

1. 使用hostnetwork=true模式
2. 在无法使用hostnetwork=true模式的情况下 需要修改源码加入dummy类型网卡

代码实现：

首先创建TAP网卡，之后使用setupIfce修改网卡ip地址

func Create() (*taptun.Tap, string){
	tap, err := taptun.NewTap(TapDeivceName)
	if err != nil {
		klog.Fatalf("%v:%v",TapDeivceName,err)
	}
	self := net.IPv4(169, 254, 2, 1)
	mask := net.IPv4Mask(255, 255, 255, 255)
	setupIfce(net.IPNet{IP: self, Mask: mask}, TapDeivceName)
	return tap,self.String()
}


func setupIfce(ipNet net.IPNet, dev string) {
	if err := exec.Command("ip", "link", "set", dev, "up").Run(); err != nil {
		klog.Fatal(err)
	}
	if err := exec.Command("ip", "addr", "add", ipNet.String(), "dev", dev).Run(); err != nil {
		klog.Fatal(err)
	}
}

 

//创建网卡
	ifce, ipaddr := tap.Create()
	tapServer := &http.Server{
		Addr:           fmt.Sprintf("%v:10261", ipaddr),
		Handler:        proxyHandler,
		MaxHeaderBytes: 1 << 20,
	}

    tapProxyServer := tapProxyServer{
    	ifce:    ifce,
    	tapHttp: tapServer,
	}

将虚拟网卡绑定hub的proxy

func NewYurtHubServer(cfg *config.YurtHubConfiguration,
	certificateMgr interfaces.YurtCertificateManager,
	proxyHandler http.Handler) Server {
	hubMux := mux.NewRouter()
	registerHandlers(hubMux, cfg, certificateMgr)
	hubServer := &http.Server{
		Addr:           cfg.YurtHubServerAddr,
		Handler:        hubMux,
		MaxHeaderBytes: 1 << 20,
	}

	proxyServer := &http.Server{
		Addr:           cfg.YurtHubProxyServerAddr,
		Handler:        proxyHandler,
		MaxHeaderBytes: 1 << 20,
	}

	//创建网卡
	ifce, ipaddr := tap.Create()
	tapServer := &http.Server{
		Addr:           fmt.Sprintf("%v:10261", ipaddr),
		Handler:        proxyHandler,
		MaxHeaderBytes: 1 << 20,
	}

    tapProxyServer := tapProxyServer{
    	ifce:    ifce,
    	tapHttp: tapServer,
	}
	return &yurtHubServer{
		hubServer:   hubServer,
		proxyServer: proxyServer,
		tapServer:   tapProxyServer,
	}
}
func (s *yurtHubServer) Run() {
	go func() {
		err := s.hubServer.ListenAndServe()
		if err != nil {
			panic(err)
		}
	}()


	go func() {
		err := s.proxyServer.ListenAndServe()
		if err != nil {
			panic(err)
		}
	}()

	klog.Infof("tap IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
	err := s.tapServer.tapHttp.ListenAndServe()
	if err != nil {
		panic(err)
	}
	klog.Infof("tap1 IP:%v ifce:%v ",s.tapServer.tapHttp.Addr,s.tapServer.ifce.String())
}