docker高级应用之单机持久化固定容器IP

转载 2015年07月08日 14:56:37

转载:http://dl528888.blog.51cto.com/2382721/1616527


一、程序介绍

使用到的程序有2个:

1、创建容器程序,名字是create_docker_container_use_static_ip.sh,使用shell脚本编写。

2、监控容器状态并自动给重启没有ip的容器设置ip,名字是auto_check_modify_container.py,使用python编写。

创建容器程序的逻辑:

1、运行创建脚本来创建docker container;

2、创建的信息会写入到etcd里进行保存;

3、根据配置信息来创建具体的container。

其中我程序使用openvswitch来坐桥接,并对不同vlan做不同的网段设置。

监控容器并给自动重启设置ip程序逻辑:

1、从etcd里获取容器的信息;

2、根据第一步的信息,进行nmap扫描ssh服务是否启动,未启动的转给下一步;

3、收到上一步的ip,然后根据etcd的容器信息进行重新pipework配置之前的固定ip。

下面是创建容器程序脚本内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#!/bin/bash
#author:Deng Lei
#email: dl528888@gmail.com
#this script is create new docker container and use pipework add static ip,when container is restart,it also use last static ip
container=$1
images=$2
daemon_program=$3
vlan_tag=$4
#ip_pool='172.16.10.1/24'
bridge=ovs2
default_vlan=1
default_vlan_ip='172.16.0.1/24'
email='244979152@qq.com'
local_ip=`/sbin/ifconfig grep 'inet'grep -Ev '(127|117|211|172|::1|fe)' |awk '{print $2}'|head -n 1`
if [ -z $1 ] || [ -z $2 ]; then
    echo "Usage: container_name container_image vlan_tag(default:1)"
    echo "Example: I want create new docker container test1 centos6:latest supervisord vlan 100"
    echo "The command is: bash `basename $0` test1 centos6:latest supervisord 100"
    exit 1
fi
[ `docker ps -a|grep -v "NAMES"|awk '{print $NF}'|grep "$container" &>>/dev/null && echo 0 || echo 1` -eq 0 ] && echo "The container $container is exist!" && exit 1
[ `which pipework &>/dev/null && echo 0|| echo 1` -eq 1 ] && echo "I can't find pipework,please install it!" && exit 1
if [ -z $3 ];then
    vlan_tag=${default_vlan}
fi
#create new docker container
docker run --restart always -d --net="none" --name="$container" $images $daemon_program &>>/dev/null
#check openvswitch bridge exist
if [ `ovs-vsctl  list-br |grep ${bridge} &>/dev/null && echo 0|| echo 1` -eq 1 ];then
    ovs-vsctl add-br ${bridge}
fi
#set up default vlan info
check_default_vlan_exist=`etcdctl get /app/docker/${local_ip}/vlan${default_vlan}/gateway 2>/dev/null || echo 1`
if [ ${check_default_vlan_exist} = "1" ];then
    ovs-vsctl add-port $bridge vlan${default_vlan} tag=${default_vlan} -- set interface vlan${default_vlan} type=internal
    #ifconfig vlan${default_vlan} ${default_vlan_ip}
    ip link set vlan${default_vlan} up
    ip addr add $default_vlan_ip dev vlan${default_vlan}
    etcdctl set /app/docker/${local_ip}/vlan${default_vlan}/nowip $default_vlan_ip &>>/dev/null
    etcdctl set /app/docker/${local_ip}/vlan${default_vlan}/gateway $default_vlan_ip &>>/dev/null
    etcdctl set /app/docker/now_vlan_ip $default_vlan_ip &>>/dev/null
fi
#set up new vlan info
new_vlan_gateway_ip=`etcdctl get /app/docker/${local_ip}/vlan${vlan_tag}/gateway 2>/dev/null || echo 1`
if [ ${new_vlan_gateway_ip} = "1" ];then
    sleep 1
    now_vlan_ip=`etcdctl get /app/docker/now_vlan_ip`
    new_vlan_ip=$(awk -v new_ip=`echo $now_vlan_ip|cut -d/ -f 1|awk -F'.' '{print $1"."$2"."$3+1"."$4}'` -v new_ip_netmask=`echo $now_vlan_ip|cut -d/ -f2` 'BEGIN{print new_ip"/"new_ip_netmask}')
    ovs-vsctl add-port $bridge vlan${vlan_tag} tag=${vlan_tag} -- set interface vlan${vlan_tag} type=internal
    ip link set vlan${vlan_tag} up
    ip addr add ${new_vlan_ip} dev vlan${vlan_tag}
#    ifconfig vlan${vlan_tag} ${new_vlan_ip}
    etcdctl set /app/docker/now_vlan_ip ${new_vlan_ip} &>>/dev/null
    etcdctl set /app/docker/${local_ip}/vlan${vlan_tag}/gateway ${new_vlan_ip} &>>/dev/null
    etcdctl set /app/docker/${local_ip}/vlan${vlan_tag}/nowip ${new_vlan_ip} &>>/dev/null
    new_vlan_gateway_ip=${new_vlan_ip}
fi
 
#calculate new ip
sleep 1
now_container_ip=`etcdctl get /app/docker/${local_ip}/vlan${vlan_tag}/nowip`
echo "now container ip is:$now_container_ip"
new_container_ip=$(awk -v new_ip=`echo $now_container_ip|cut -d/ -f 1|awk -F'.' '{print $1"."$2"."$3"."$4+1}'` -v new_ip_netmask=`echo $now_container_ip|cut -d/ -f2` 'BEGIN{print new_ip"/"new_ip_netmask}')
etcdctl set /app/docker/${local_ip}/vlan${vlan_tag}/nowip $new_container_ip &>/dev/null
echo "finish calculate new ip!"
create_time=`date +"%Y-%m-%d %T"`
`which pipework` $bridge $container ${new_container_ip}@`echo ${new_vlan_gateway_ip}|awk -F '/' '{print $1}'` @${vlan_tag}
etcdctl set /app/docker/${local_ip}/vlan${vlan_tag}/container/${container} "{'Physics_ip':'${local_ip}','Container_name':'${container}','Container_ip':'${new_container_ip}','Container_vlan':'${vlan_tag}','Container_vlan_gateway':'${new_vlan_gateway_ip}','Container_create':'${create_time}','Container_status':'running'}"

注意:如果想使用本程序,客户端必须安装openvswitch、etcd与etcdctl、pipework。

监控容器并给自动重启设置ip程序脚本内容:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/usr/bin/env python
#author:Deng Lei
#email: dl528888@gmail.com
import sys
import nmap
import multiprocessing
import time
import socket, struct, fcntl
import etcd
import re
from docker import Client
import subprocess
def get_local_ip(iface = 'em1'):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sockfd = sock.fileno()
    SIOCGIFADDR = 0x8915
    ifreq = struct.pack('16sH14s', iface, socket.AF_INET, '\x00'*14)
    try:
    res = fcntl.ioctl(sockfd, SIOCGIFADDR, ifreq)
    except:
    return None
    ip = struct.unpack('16sH2x4s8x', res)[2]
    return socket.inet_ntoa(ip)
def get_etcd_info(key):
    etcd_result=[]
    r = etcd_client.read('%s%s'%(docker_etcd_key,local_ip), recursive=True, sorted=True)
    for child in r.children:
        if child.dir is not True and "Container_name" in child.value:
            etcd_result.append(child.value)
    return etcd_result
def check_by_nmap(ip,port):
    check_result=len(nm.scan('%s'%ip,'%s'%port)['scan'])
    if check_result == 0:
        msg='%s'%(ip)
        return msg
def docker_container_stop():
    docker_container=docker_client.containers(all=True)
    container_name=[]
    container_stop_name=[]
    for in docker_container:
        if re.match('Exited',i['Status']):
            container_name.append(i['Names'])
    for in container_name:
        for in b:
        container_stop_name.append(c)
    return container_stop_name
def docker_container_all():
    docker_container=docker_client.containers(all=True)
    container_name=[]
    container_stop_name=[]
    for in docker_container:
    container_name.append(i['Names'])
    for in container_name:
        for in b:
            container_stop_name.append(c)
    return container_stop_name
 
if __name__ == "__main__":
    nm = nmap.PortScanner()
    etcd_client=etcd.Client(host='127.0.0.1', port=4001)
    docker_client = Client(base_url='unix://var/run/docker.sock', version='1.15', timeout=10)
    local_ip=get_local_ip('ovs1')
    bridge='ovs2'
    docker_etcd_key='/app/docker/'
    port='22'
    ip_list=[]
    now_etcd_result=get_etcd_info(docker_etcd_key)
    docker_container_stop_name=docker_container_stop()
    docker_container_all_name=docker_container_all()
    for in range(len(now_etcd_result)):
        now_info=eval(now_etcd_result[i])
    key='%s%s/vlan%s/container/%s'%(docker_etcd_key,local_ip,now_info['Container_vlan'],now_info['Container_name'])
        if '/'+now_info['Container_name'] not in docker_container_stop_name and now_info['Container_status'] != 'delete':
        ip_list.append(eval(now_etcd_result[i])['Container_ip'].split('/')[0])
         if now_info['Container_status'] == 'stop':
           now_info['Container_status']='running'
        etcd_client.write(key,now_info)
        else:
            now_info['Container_status']='stop'
        etcd_client.write(key,now_info)
    if '/'+now_info['Container_name'] not in docker_container_all_name:
        print '%s %s will delete,because docker client had deleted!'%(time.strftime('%Y-%m-%d %T'),key)
            now_info['Container_status']='delete'
        etcd_client.write(key,now_info)
    ip_list=list(set(ip_list))
    if len(ip_list) == 1:
        num=1
    elif len(ip_list)>=4 and len(ip_list)<=8:
        num=4
    elif len(ip_list) >8 and len(ip_list)<=20:
        num=8
    else:
        num=15
    pool = multiprocessing.Pool(processes=num)
    scan_result=[]
    for in ip_list:
        pool.apply_async(check_by_nmap, (i,port, ))
        scan_result.append(pool.apply_async(check_by_nmap, (i,port, )))
    pool.close()
    pool.join()
    result=[]
    for res in scan_result:
        if res.get() is not None:
            result.append(res.get())
    result=list(set(result))
    if len(result) <= 0:
        print '%s Everything is all right!'%time.strftime('%Y-%m-%d %T')
        sys.exit(1)
    else:
        print '%s container need modify container static ip is:%s'%(time.strftime('%Y-%m-%d %T'),result)
    for in range(len(now_etcd_result)):
        now_info=eval(now_etcd_result[i])
        if (now_info['Container_ip']).split('/')[0] in result:
        modify_container=subprocess.Popen("`which pipework` %s %s %s@%s @%s &>>/dev/null && echo 0 || echo 1"%(bridge,now_info['Container_name'],now_info['Container_ip'],now_info['Container_vlan_gateway'].split('/')[0],now_info['Container_vlan']),shell=True,stdout=subprocess.PIPE)
        if (modify_container.stdout.readlines()[0]).strip('\n') == 0:
              print 'container_name:%s container_ip:%s modify static ip is  fail!'%(now_info['Container_name'],now_info['Container_ip'])
        else:
            print 'container_name:%s container_ip:%s modify static ip is success!'%(now_info['Container_name'],now_info['Container_ip'])

注意:使用本脚本必须安装nmap、etcd、docker模块。

其中python里nmap的安装方法是

1
easy_install python-nmap

其中python里etcd的安装方法是

安装基础库

1
yum install libffi libffi-devel python-devel

安装程序

1
2
3
git clone https://github.com/jplana/python-etcd.git
cd python-etcd
python setup.py install

其中python里docker的安装方法是

1
easy_install docker-py

二、效果展示

需求:

创建主机test1、test2、test3、test4,需要test1与test2属于vlan100,test3与test4属于vlan300,架构图如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
                ovs2
                  |
                  |
            -----------------   
            |               |
            |               |
vlan100(tag100)         vlan300(tag300)
      |                             |
      |                             |
  -----------              ----------------------      
  |         |               |                  | 
  |         |               |                  |
test1      test2           test3             test4
(tag100)   (tag100)       (tag300)          (tag300)

下面是操作

1、创建ovs1、ovs2网桥,其中em1绑定到ovs1走内网,ovs2是容器的网络

1
2
3
4
5
6
ovs-vsctl add-br ovs1
ovs-vsctl add-br ovs2
ovs-vsctl add-port ovs1 em1
ip link set ovs1 up
ifconfig em1 0
ifconfig ovs1 10.10.17.3

2、创建test1、test2、test3、test4的容器

1
2
3
4
5
6
7
8
[root@docker-test3 ~]# sh create_docker_container_use_static_ip.sh test1 docker.ops-chukong.com:5000/centos6-http:new /usr/bin/supervisord 100
{'Physics_ip':'10.10.17.3','Container_name':'test1','Container_ip':'172.16.1.2/24','Container_vlan':'100','Container_vlan_gateway':'172.16.1.1/24','Container_create':'2015-03-02 14:59:59','Container_status':'running'}
[root@docker-test3 ~]# sh create_docker_container_use_static_ip.sh test2 docker.ops-chukong.com:5000/centos6-http:new /usr/bin/supervisord 100
{'Physics_ip':'10.10.17.3','Container_name':'test2','Container_ip':'172.16.1.3/24','Container_vlan':'100','Container_vlan_gateway':'172.16.1.1/24','Container_create':'2015-03-02 15:00:09','Container_status':'running'}
[root@docker-test3 ~]# sh create_docker_container_use_static_ip.sh test3 docker.ops-chukong.com:5000/centos6-http:new /usr/bin/supervisord 300
{'Physics_ip':'10.10.17.3','Container_name':'test3','Container_ip':'172.16.2.2/24','Container_vlan':'300','Container_vlan_gateway':'172.16.2.1/24','Container_create':'2015-03-02 15:00:22','Container_status':'running'}
[root@docker-test3 ~]# sh create_docker_container_use_static_ip.sh test4 docker.ops-chukong.com:5000/centos6-http:new /usr/bin/supervisord 300
{'Physics_ip':'10.10.17.3','Container_name':'test4','Container_ip':'172.16.2.3/24','Container_vlan':'300','Container_vlan_gateway':'172.16.2.1/24','Container_create':'2015-03-02 15:00:31','Container_status':'running'}

这个脚本第一个参数是创建的容器名,第二个参数是docker images,第三个参数是运行命令,第四个参数是vlan。

下面是当前docker信息

1
2
3
4
5
6
[root@docker-test3 ~]# docker ps -a
CONTAINER ID        IMAGE                                          COMMAND                CREATED              STATUS              PORTS               NAMES
386b4180b3d0        docker.ops-chukong.com:5000/centos6-http:new   "/usr/bin/supervisor   About a minute ago   Up About a minute                       test4
92fee54ec498        docker.ops-chukong.com:5000/centos6-http:new   "/usr/bin/supervisor   About a minute ago   Up About a minute                       test3
4dc84301894a        docker.ops-chukong.com:5000/centos6-http:new   "/usr/bin/supervisor   About a minute ago   Up About a minute                       test2
45d4a73c6ed0        docker.ops-chukong.com:5000/centos6-http:new   "/usr/bin/supervisor   2 minutes ago        Up 2 minutes                            test1

下面是openvswitch的信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
[root@docker-test3 ~]# ovs-vsctl show
d895d78b-8c89-49bc-b429-da6a4a2dcb3a
    Bridge "ovs2"
        Port "vlan300"
            tag: 300
            Interface "vlan300"
                type: internal
        Port "ovs2"
            Interface "ovs2"
                type: internal
        Port "veth1pl11491"
            tag: 100
            Interface "veth1pl11491"
        Port "veth1pl12003"
            tag: 300
            Interface "veth1pl12003"
        Port "vlan100"
            tag: 100
            Interface "vlan100"
                type: internal
        Port "veth1pl12251"
            tag: 300
            Interface "veth1pl12251"
        Port "veth1pl11788"
            tag: 100
            Interface "veth1pl11788"
        Port "vlan1"
            tag: 1
            Interface "vlan1"
                type: internal
    Bridge "ovs1"
        Port "em1"
            Interface "em1"
        Port "ovs1"
            Interface "ovs1"
                type: internal
    ovs_version: "2.3.1"

下面是etcd的信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
[root@docker-test3 ~]# etcdctl ls / --recursive|grep docker|grep "10.10.17.3/"
/app/docker/ip/vlan/10.10.17.3/gate_way
/app/docker/ip/vlan/10.10.17.3/now_ip
/app/docker/10.10.17.3/vlan300
/app/docker/10.10.17.3/vlan300/container
/app/docker/10.10.17.3/vlan300/container/test3
/app/docker/10.10.17.3/vlan300/container/test4
/app/docker/10.10.17.3/vlan300/gateway
/app/docker/10.10.17.3/vlan300/nowip
/app/docker/10.10.17.3/vlan1
/app/docker/10.10.17.3/vlan1/gateway
/app/docker/10.10.17.3/vlan1/nowip
/app/docker/10.10.17.3/vlan100
/app/docker/10.10.17.3/vlan100/gateway
/app/docker/10.10.17.3/vlan100/nowip
/app/docker/10.10.17.3/vlan100/container
/app/docker/10.10.17.3/vlan100/container/test1
/app/docker/10.10.17.3/vlan100/container/test2

查看test1的信息

1
2
[root@docker-test3 ~]# etcdctl get /app/docker/10.10.17.3/vlan100/container/test1
{'Physics_ip':'10.10.17.3','Container_name':'test1','Container_ip':'172.16.1.2/24','Container_vlan':'100','Container_vlan_gateway':'172.16.1.1/24','Container_create':'2015-03-02 14:59:59','Container_status'