在Solaris下使用IPMP（IP Multipathing）

在Solaris下使用IPMP（IP Multipathing）

为什么要使用IPMP

1、为了增加网络的吞吐量，实现多网卡的负载均衡。
2、为了实现网卡冗余，提高系统可靠性与可用性。

比如有某块网卡因为某些不可预测的原因，从网络上断开了，这时候客户就再也访问不了与该网卡相关联的IP地址了。但是如果使用了IPMP，我们就可以把几块网卡配置成一个组，一旦组里面的某块网卡与网络的连接断开了，那么IPMP就会检测到这个错误，然后自动进行故障转移。IPMP会保持该网卡上己存在的连接，并把发往该网卡的数据转发到组里其它正常运行的网卡，再由该网卡发送到网络上。从而保证网络的持续可用。同时IPMP会自动把负载过大的网卡上的负载摊铺到组里其它的网卡上，从而达到网卡的负载均衡。

配置IPMP

IPMP的配置有点烦，主要是因为它的概念不易理解，IPMP要求每个网卡至少要配置有两个IP，一个是网卡的实际IP地址，叫Data address；一个是用来监控网卡的运行状态，错误检测，叫Test address。当检测到故障时，IPMP会把Data address从故障网卡上转移到组里正常运行的网卡上。外部应用程序应该使用Data address，避免使用Test address。

IPMP对网络配置和拓扑有如下强制要求：
1、IPMP组里的所有网络接口必须要有一个唯一的MAC地址。
   注意，默认情况下，在基于SPARC架构的系统上，所有的网络接口共享一个MAC地址。因此，为了在基于SPARC架构的系统上使用IPMP，   你必须修改默认的MAC地址。
2、IPMP组里的所有网络接口必须是同一类型的媒介。
3、IPMP组里的所有网络接品必须在同一IP链路层上。注：即在同一网段内。
4、你可能使用特定类型的网络接口或者给每一个网络接口配置额外的IP地址，这依赖于你对故障探测的需求。

因为整个配置过程比较烦，如果每次都手工从头配置的话，容易出错，所以我把这些过程写成了脚本，需要的时候调出来用就可以了。下面的脚本是我去年写的，一个是shell脚本，一个是用Python+Tkinter写的。

SHELL脚本（mpathconfig.sh）

# !/ bin / sh
#
#    configure the ip multipath network
#

echo  " Please input the number of networkd cards(1 ~ 4): "
read noc  #number of network cards

while  [ $noc  - le  0   - o $noc  - ge  5  ]
do
    echo  " Illegal number of cards, need 1 ~ 4 "
    echo  " Please input the number of networkd cards "
    read noc  #number of network cards
done

hostfile =/ etc / inet / hosts
ipfile =/ etc / inet / ipnodes

if  [  !   - w  / etc / inet / hosts ]
then
    chmod u + w $hostfile 
fi

cp . / rdisc  / etc / init.d
chmod u + x  / etc / init.d / rdisc
ln  / etc / init.d / rdisc  / etc / rc2.d / S70rdisc

mv $hostfile ${hostfile}.bak
echo  " 127.0.0.1    localhost "   >  $hostfile

mv $ipfile ${ipfile}.bak
echo  " ::1    localhost "   >  $ipfile

echo  " Please input your group name "
read group_name

touch ${group_name}.sh
echo  " #!/bin/sh "   >  ${group_name}.sh
chmod u + x ${group_name}.sh

answer = yes
n = 1
while  [ $n  - le $noc ]
do
     if  [ $answer  =   " yes "   - o $answer  =   " YES "   - o $answer  =   " y "   - o $answer  =   " Y "  ]
    then    
        echo  " Please input the system name "
        read sys_name
        echo  " Please input the manage address for interface "
        read mng_addr
        echo  " $mng_addr    $sys_name    loghost "   >>  $hostfile
        echo  " $mng_addr    $sys_name    loghost "   >>  $ipfile
    fi
    
    echo  " Please input the logical interface name $n "
    read int_name
    echo  " Please input the test address for interface $n "
    read test_addr

    echo  " $test_addr    gdlc-${int_name} "   >>  $hostfile
    echo  " $test_addr    gdlc-${int_name} "   >>  $ipfile

     if  [  - w  / etc / hostname.${int_name} ]
    then
        mv  / etc / hostname.${int_name}  / etc / hostname.${int_name}.bak
    fi

    touch  / etc / hostname.${int_name}
    
    trap  " mv /etc/hostname.${int_name}.bak /etc/hostname.${int_name};mv ${hostfile}.bak ${hostfile}; mv ${ipfile}.bak ${ipfile} "  INT TERM
    
    echo  " gdlc-${int_name} netmask + broadcast + group $group_name deprecated -failover up "   >   / etc / hostname.${int_name}
    echo  " addif $sys_name netmask + broadcast + failover up "   >>   / etc / hostname.${int_name}
    
    echo  " ifconfig $int_name group $group_name "   >>  ${group_name}.sh
    ifconfig $int_name plumb gdlc - ${int_name} netmask  +  broadcast  +   - failover deprecated up
    ifconfig $int_name addif $sys_name netmask  +  broadcast  +  failover up

    n = `expr $n  +   1 `
    
    echo  " Do you want to input more manage IP? "
    read answer
done

echo  " Is the host use for router? "
read answer

if  [ $answer  =   " yes "   - o $answer  =   " YES "   - o $answer  =   " y "   - o $answer  =   " Y "  ]
then
    touch  / etc / notrouter
    echo  " Create file /etc/notrouter "
fi

. . / ${group_name}.sh
rm ${group_name}.sh

echo  " done "

 与shell脚本相应的rdisc文件

# !/ bin / sh
case   " $1 "   in
' start ' )
     if  [  - x  / usr / bin / pgrep ]
    then
         / usr / bin / pgrep  - x  - u  0   in .rdisc  >   / dev / null   2 >& 1   ||  
            / usr / sbin / in .rdisc  - f  >  dev / msglog  2 >& 1
     else
        logger Cannot execute  / usr / bin / pgrep,  in .rdisc not started.
    fi
    ;;
' stop ' )
     / usr / bin / pkill  - x  - u  0   in .rdisc
    ;;
* )
    echo  " Usage:$0 {start | stop} "
    ;;
esac
exit  0

Python脚本(mpath.py)

# !/usr/bin/python

import  Tkinter
import  os
import  sys
import  Dialog

import  sysinfo

class  MultiPathConfig(Tkinter.Frame):
     def   __init__ (self, master  =  None):
        Tkinter.Frame. __init__ (self, master)
        self.si  =  sysinfo.SysInfo()
        self.iname_list  =  self.si.get_iname_list()
         if  len(self.iname_list)  <   3  :
            self.iname_count  =   3
         else :
            self.iname_count  =  len(self.iname_list)
        self.sys_name  =   " sys- "
        self.int_name  =   " gdlc- "
        self.column_count  =   4
        self.other_row_count  =   1
        
        self.empty_ip_address  =   - 1
        self.illegal_ip_address  =   - 2
    
        self.test_ip_list  =  []
        self.man_ip_list  =  []
        self.check_button_list  =  []
        self.check_value_list  =  []
        self.flag  =  True

         for  name  in  self.iname_list:
            self.check_value_list.append(Tkinter.IntVar(0))

        self.create_widgets()
    
     def  create_widgets(self):
        self.grid(sticky  =  Tkinter.N + Tkinter.W + Tkinter.E + Tkinter.S)
        self.top  =  self.winfo_toplevel()
        self.top.rowconfigure(0, weight  =   1 )
        self.top.columnconfigure(0, weight  =   1 )
    
         # Make all columns stretchable
         for  i  in  range(self.column_count):
            self.columnconfigure(i, weight  =   1 )

        self.iname_lbl  =  Tkinter.Label(self, text  =   " Name " )
        self.iname_lbl.grid(row  =  0, column  =  0, 
            sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )
    
        self.test_ip_addr_lbl  =  Tkinter.Label(self,
            text  =   " Test IP Address " )
        self.test_ip_addr_lbl.grid(row  =  0, column  =   1 ,
            sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

         for  i  in  range(len(self.iname_list)):
            self.iname  =  Tkinter.Label(self, text  =  self.iname_list[i])
            self.iname.grid(row  =  i + self.other_row_count, column  =  0, 
                sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

            self.test_ip  =  Tkinter.Entry(self, bg = " #ffffff " )
            self.test_ip.grid(row  =  i + self.other_row_count, column  =   1 , 
                sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )
            self.test_ip_list.append(self.test_ip)
    
            self.cb_var  =  Tkinter.IntVar()
            self.check_button  =  Tkinter.Checkbutton(self, text  =   " select " ,
                variable  =  self.cb_var, 
                onvalue  =   1 , offvalue  =  0)
            self.check_value_list[i]  =  self.cb_var
            self.check_button.var  =  self.check_value_list[i]
            self.check_button.grid(row  =  i + self.other_row_count, column  =   2 , 
                sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )
            self.check_button_list.append(self.check_button)

        self.add_button(self, 0,  3 ,  " Save " , self.on_save)
        self.add_button(self,  1 ,  3 ,     " OK " , self.on_ok)
        self.add_button(self,  2 ,  3 ,  " Apply " , self.on_apply)
        self.add_button(self,  3 ,  3 ,     " Close " , self.on_close)

        self.label  =  Tkinter.Label(self, text  =   " Group Name " )
        self.label.grid(row  =  self.iname_count  +  self.other_row_count, column  =  0,
            sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )
        self.group_name  =  Tkinter.Entry(self, bg = " #ffffff " )
        self.group_name.grid(row  =  self.iname_count  +  self.other_row_count, column  =   1 ,
            sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

        self.other_row_count  +=   1
        self.man_ip_addr_lbl  =  Tkinter.Label(self, text  =   " Manage IP  "   +  str( 1 ))
        self.man_ip_addr_lbl.grid(row  =  self.iname_count  +  self.other_row_count, column  =  0,
            sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

        self.man_ip  =  Tkinter.Entry(self, bg = " #ffffff " )
        self.man_ip.grid(row  =  self.iname_count  +  self.other_row_count, column  =   1 , 
            stick  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )
        self.man_ip_list.append(self.man_ip)
    
        self.add_button(self, self.iname_count  +  self.other_row_count,  3 ,
             " More " , self.on_more)
    
         # Make all rows stretchable
         for  i  in  range(self.iname_count  +  len(self.man_ip_list)  +  self.other_row_count):
            self.rowconfigure(i, weight  =   1 )
    
     def  add_button(self, root, row, column, text, command):
        self.button  =  Tkinter.Button(root, text  =  text, command  =  command)
        self.button.grid(row  =  row, column  =  column, 
                sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

     # Save configure to file
     def  on_save(self):
        self.check_all()
         for  iname  in  self.iname_list:
             if  self.check_value_list[self.iname_list.index(iname)].get()  ==   1 :
                 try :
                    os.rename( " /etc/hostname. "   +  iname,  " /etc/hostname. "   +  iname  +   " .bak " )
                 except  OSError:
                     print   " No such file: /etc/hostname. "   +  iname,  "  didn't backup. "
                
                cmd_str  =  self.int_name  +  iname 
                     +   "  netmask + broadcast + group  "   +  self.group_name.get() 
                     +   "  deprecated -failover up "  
                 if  self.flag  ==  True:
                    cmd_str  +=   " addif  "   +  os.popen( " hostname " ).read()[0: - 1 ] 
                         +   "  netmask + broadcast + failover up "
                    self.flag  =  False
                 else :
                    cmd_str  +=   " addif  "   +  self.sys_name  +  iname 
                         +   "  netmask + broadcast + failover up "
                 try :
                    file  =  open( " /etc/hostname. "   +  iname,  " w " )
                    file.write(cmd_str)
                    file.close()
                 except  IOError:
                     print   " Can not open file: /etc/hostname. "   +  iname
        self.flag  =  True
    
         # Save test ip address to /etc/inet/hosts and /etc/inet/ipnodes
        hostfile  =   " /etc/inet/hosts "
        ipfile  =   " /etc/inet/ipnodes "
         try :
            os.rename(hostfile, hostfile  +   " .bak " )
            host_file  =  open(hostfile,  " a+ " )
            host_file.write( " 127.0.0.1 " )
            os.rename(ipfile, ipfile  +   " .bak " )
            ip_file  =  open(ipfile,  " a+ " )
            ip_file.write( " ::  localhost " )
         except  OSError:
             print   " Can not rename file  "   +  hostfile  +   "  to  "   +  hostfile  +   " .bak "
             print   " Or can not rename file  "   +  ipfile  +   "  to  "   +  ipfile  +   " .bak "
             return
         except  IOError:
             print   " Can not open file:  "   +  hostfile
             print   " Or can not open file:  "   +  ipfile
             return
         else :
             for  iname  in  self.iname_list:
                 if  self.check_value_list[self.iname_list.index(iname)].get()  ==   1 :
                    host_file.write(self.test_ip_list[self.iname_list.index(iname)].get()
                         +   " "   +  self.int_name  +  iname  +   " " )
                    ip_file.write(self.test_ip_list[self.iname_list.index(iname)].get()
                         +   " "   +  self.int_name  +  iname  +   " " )

             for  ip  in  self.man_ip_list:
                 if  self.man_ip_list.index(ip)  ==  0:
                    host_file.write(ip.get()  +   " "  
                         +  self.si.get_hostname()  +   " loghost " )
                    ip_file.write(ip.get()  +   " "  
                         +  self.si.get_hostname()  +   " loghost " )
                 else :
                    host_file.write(ip.get()  +   " "   +  self.sys_name 
                         +  self.iname_list[self.man_ip_list.index(ip)]  +   " loghost " )
                    ip_file.write(ip.get()  +   " "   +  self.sys_name 
                         +  self.iname_list[self.man_ip_list.index(ip)]  +   " loghost " )
            ip_file.close()
            host_file.close()

     def  on_ok(self):
        self.on_save()
        self.on_apply()
    
     # Apply configure immediately but not save.
     def  on_apply(self):
        self.check_all()
         for  iname  in  self.iname_list:
             if  self.check_value_list[self.iname_list.index(iname)].get()  ==   1 :
                cmd  =   " ifconfig  "   +  iname  +   "  group  "   +  self.group_name.get()
                ret  =  os.system(cmd)
         for  iname  in  self.iname_list:
             if  self.check_value_list[self.iname_list.index(iname)].get()  ==   1 :
                cmd  =   " ifconfig  "   +  iname
                cmd  +=   "  netmask + broadcast + -failover deprecated up "
                ret  =  os.system(cmd)
                 if  self.flag  ==  True 
                     or  self.iname_list.index(iname)  >=  len(self.man_ip_list):
                    cmd  =   " ifconfig  "   +  iname  +   "  addif  "   +  self.si.get_hostname()
                    self.flag  =  False
                 elif  self.iname_list.index(iname)  >=  len(self.man_ip_list):
                    cmd  =   " ifconfig  "   +  iname  +   "  addif  "   +  self.si.get_hostname() 
                 else :
                    cmd  =   " ifconfig  "   +  iname  +   "  addif  "   +  self.sys_name  +  iname
                cmd  +=   "  netmask + broadcast + failover up "
                ret  =  os.system(cmd)
        self.flag  =  True
        
     def  on_close(self):
        self.quit()

     def  on_more(self):
         if  len(self.man_ip_list)  >=  self.get_enable_count():
             return

         # Make new line stretchable
        self.rowconfigure(self.iname_count  +  len(self.man_ip_list) 
             +  self.other_row_count, weight  =   1 )
    
        self.man_ip_addr_lbl  =  Tkinter.Label(self,
            text  =   " Manage IP  "   +  str(len(self.man_ip_list)  +   1 ))
        self.man_ip_addr_lbl.grid(row  =  self.iname_count  +  len(self.man_ip_list)
             +  self.other_row_count, 
            column  =  0, sticky  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

        self.man_ip  =  Tkinter.Entry(self, bg = " #ffffff " )
        self.man_ip.grid(row  =  self.iname_count  +  len(self.man_ip_list)
             +  self.other_row_count, 
            column  =   1 , stick  =  Tkinter.E  +  Tkinter.W, padx  =   5 , pady  =   5 )

        self.man_ip_list.append(self.man_ip)
        
     def  message_box(self, title, text, strings, default):
        dlg  =  Dialog.Dialog(None, title  =  title, text  =  text, default  =  default,
            bitmap  =  Dialog.DIALOG_ICON, strings  =  strings)
         return  dlg.num

     def  check_all(self):
         if  self.check_all_test_ip()  ==  self.empty_ip_address:
            ret  =  self.message_box( " Message " , 
                 " Get an empty test IP address, back to provide one or quit? " ,
                ( " back " ,  " quit " ), 0)
             if  ret  ==  0: return
             else : self.quit()
         elif  self.check_all_test_ip()  ==  self.illegal_ip_address:
            ret  =  self.message_box( " Message " ,
                 " Get an illegal test IP address, back to modify or quit? " ,
                ( " back " ,  " quit " ), 0)
             if  ret  ==  0:  return
             else : self.quit()
         if  self.check_all_man_ip()  ==  self.empty_ip_address:
            ret  =  self.message_box( " Message " ,
                 " Get an empty manage IP address, back to provide one or quit? " ,
                ( " back " ,  " quit " ), 0)
             if  ret  ==  0:  return
             else : self.quit()
         elif  self.check_all_man_ip()  ==  self.illegal_ip_address:
            ret  =  self.message_box( " Message " ,
                 " Get an illegal manage IP address, back to modify or quit? " ,
                ( " back " ,  " quit " ), 0)
             if  ret  ==  0:  return
             else : self.quit()
         if  self.check_group_name()  ==   - 1 :
            ret  =  self.message_box( " Message " , 
                 " Get an empty group name, back to provide one or quit? " , 
                ( " back " ,  " quit " ), 0)
             if  ret  ==  0:  return
             else : self.quit()

     def  check_group_name(self):
         if  self.group_name.get()  ==   "" :
             return   - 1
         return  0
        
     def  check_ip(self, ip):
         if  ip  ==   "" :
             return  self.empty_ip_address
        ipnum  =  str(ip).split( " . " )
         if  len(ipnum)  !=   4 :
             return  self.illegal_ip_address
         for  n  in  ipnum:
             if   not  n.isdigit()  or  int(n)  >   255   or  int(n)  <  0:
                 return  self.illegal_ip_address
         return  0

     def  check_all_test_ip(self):
         for  i  in  range(len(self.iname_list)):
             if  self.check_value_list[i].get()  ==   1 :
                ret  =  self.check_ip(self.test_ip_list[i].get())
                 if  ret  ==  self.empty_ip_address:
                     return  ret
                 elif  ret  ==  self.illegal_ip_address:
                     return  ret
         return  0
    
     def  check_all_man_ip(self):
         for  i  in  range(len(self.man_ip_list)):
            ret  =  self.check_ip(self.man_ip_list[i].get())
             if  ret  ==  self.empty_ip_address:
                 return  ret
             elif  ret  ==  self.illegal_ip_address:
                 return  ret
         return  0

     def  get_enable_count(self):
        count  =  0
         for  var  in  self.check_value_list:
             if  var.get()  ==   1 :
                count  +=   1
         return  count
        
def  main():
    si  =  sysinfo.SysInfo()
     if  si.check_user()  ==   - 1 :
         return   - 1
    si.make_rdisc()
    main_form  =  MultiPathConfig()
    main_form.master.title( " IP Network Maltipath Configure " )
    Tkinter.mainloop()

if   __name__   ==   " __main__ " :
    main()

Python脚本sysinfo.py文件

# !/usr/bin/python

import  os
import  sys

class  SysInfo:
     def  check_user(self):
        uid  =  os.popen( " id -u " ).readline()[0: - 1 ]
         if  uid  !=   " 0 " :
             print   " Need user root to run this program "
             return   - 1

     def  get_hostname(self):
        host_name  =  os.popen( " hostname " ).readline()[0: - 1 ]
         return  host_name
        
     def  get_iname_list(self):
         # The next two lines use for test.
         if  os.popen( " uname " ).readline()[0: - 1 ]  ==   " Linux " :
             return  [ " eth0 " ,  " eth1 " ,  " eth2 " ]
        self.iname_list  =  []
        self.inames  =  os.popen( " ifconfig -a | cut -d " " -f 1 " ).readlines()
         for  name  in  self.inames:
             if  name  !=   ' '   and  name[0: 4 ]  !=   ' lo0: ' :
                index  =  name.find( " : " )
                 if  index  !=   - 1 :
                     if  name.find( " : " , index + 1 )  !=   - 1 :   # logical interface
                         continue
                    name  =  name[0:index]  
                    self.iname_list.append(name)
         return  self.iname_list
        
     def  make_rdisc(self):
        text  =   """
#!/bin/sh
case "$1" in
'start')
    if [ -x /usr/bin/pgrep ]
    then
        /usr/bin/pgrep -x -u 0 in.rdisc > /dev/null 2>&1 || /usr/sbin/in.rdisc -f > /dev/msglog 2>&1
    else
        logger Cannot execute /usr/bin/pgrep, in.rdisc not started.
    fi
    ;;
'stop')
    /usr/bin/pkill -x -u 0 in.rdisc
    ;;
*)
    echo "Usage:$0 {start | stop}"
    ;;
esac
exit 0
         """
         try :
            file  =  open( " /etc/init.d/rdisc " ,  " w " )
            file.write(text)
            file.close()
         except  IOError:
             print   " Can not open file /etc/init.d/rdisc "
             return   - 1
        os.system( " chmod u+x /etc/init.d/rdisc " )
        os.system( " ln /etc/init.d/rdisc /etc/rc2.d/S70rdisc " )

Python脚本用了Tkinter做GUI界面，使用起来比shell脚本方便点。