淘宝服务器架构框架图，简单实现....

最新推荐文章于 2019-12-19 12:00:27 发布

Stephen@Tang

最新推荐文章于 2019-12-19 12:00:27 发布

阅读量2.7k

点赞数

分类专栏：会话保持复制共享 LVS HAProxy HA

HA 同时被 3 个专栏收录

34 篇文章 0 订阅

订阅专栏

LVS

33 篇文章 0 订阅

订阅专栏

HAProxy

8 篇文章 0 订阅

订阅专栏

本文出自 “峰云，就她了。” 博客，请务必保留此出处http://rfyiamcool.blog.51cto.com/1030776/887983

这几天闲着没事从老男孩老师看到一个淘宝网的框架图，挺感兴趣的，cdn和集群线上的架构我都做过，但是没有接触过这么大的环境，先简单的实现看看, 当然了真正的淘宝架构肯定不能像我这样的，但是自己过过实验瘾也挺爽的。

陆续的把脚本贴出来。。。

脚本有不严谨的地方，请大家指出。。。。。

脚本的ip貌似和图上都对不上，自己修改和增加吧~~~~~~~~~

说实话，lvs配置是最简单，没什么好配置的，集群环境我用lvs较少，因为没有正则的功能，当然了他作为4层的东西，优势在于大流量的承载转发。

  
  mkdir /usr/local/src/lvs   
cd /usr/local/src/lvs   
wget http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.24.tar.gz   
wget http://www.keepalived.org/software/keepalived-1.1.15.tar.gz  
lsmod |grep ip_vs  
uname -r  
ln -s /usr/src/kernels/$(uname -r)/usr/src/linux  
tar zxvf ipvsadm-1.24.tar.gz  
cd ipvsadm-1.24  
make && make install  
tar zxvf keepalived-1.1.15.tar.gz  
cd keepalived-1.1.15  
./configure&& make && make install  
cp /usr/local/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/  
cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/  
mkdir /etc/keepalived  
cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/  
cp /usr/local/sbin/keepalived /usr/sbin/  
#you can service keepalived start|stop  
#master  
cat >> /usr/local/etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived  
global_defs {  
   notification_email {  
        rfyiamcool@163.com  
   }  
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1  
   router_id LVS_DEVEL  
}  
vrrp_instance VI_1 {  
    state MASTER   # other backup
    interface eth0  
    virtual_router_id 51  
    priority 100    #  other 90
    advert_int 1  
    authentication {  
        auth_type PASS  
        auth_pass 1111  
    }  
    virtual_ipaddress {  
        10.10.10.88  
    }  
}  
virtual_server 10.10.10.88 80 {  
    delay_loop 6  
    lb_algo rr  
    lb_kind DR  
    persistence_timeout 50  
    protocol TCP  
    real_server 10.10.10.21 80 {  
        weight 3  
        TCP_CHECK {  
        connect_timeout 10  
        nb_get_retry 3  
        delay_before_retry 3  
        connect_port 80  
        }  
    }  
    real_server 10.10.10.22 80 {  
        weight 3  
        TCP_CHECK {  
        connect_timeout 10  
        nb_get_retry 3  
        delay_before_retry 3  
        connect_port 80  
        }  
    }  
}  
EOF  
service keepalived start

咱们先把二层的haproxy搞定，ip什么的大家自己改吧。

  
  #!/bin/bash  
cd /usr/local/src/  
wget http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.8.tar.gz  
tar zxf haproxy-1.4.8.tar.gz  
cd haproxy-1.4.8  
uname -a  
make TARGET=linux26PREFIX=/usr/local/haproxy  
make install PREFIX=/usr/local/haproxy  
cat >> /usr/local/haproxy/haproxy.cfg <<EOF
global  
        log 127.0.0.1   local0     ###全局日志  
        maxconn 4096           ###最大连接数  
        chroot /usr/local/haproxy  
        uid 501      ###用户ID  
        gid 501      ###组ID  
        daemon     ###后台运行  
        nbproc 1             ###创建进程数  
        pidfile /usr/local/haproxy/haproxy.pid      ###pid文件  
defaults  
        log     127.0.0.1       local3  
        mode    http           ###支持的模式  
        option httplog            ###日志格式  
        option httpclose        ###请求完成后关闭http通道  
        option dontlognull  
        option forwardfor     ###apache日志转发  
        option redispatch  
        retries 2             ###重连次数  
        maxconn 2000  
        balance roundrobin            ###算法类型  
        stats   uri     /haproxy-stats      ###状态统计页面  
        #stats   auth   admin:admin       ###状态统计页面用户名密码，可选  
        contimeout      5000            ###连接超时  
        clitimeout      50000             ###客户端超时  
        srvtimeout      50000          ###服务器超时  
listen   proxy *:80          ###访问地址及端口  
        option httpchk HEAD /index.html  HTTP/1.0           ###健康检查页面  
    server web2 10.10.10.30:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
    server web2 10.10.10.31:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
    server web2 10.10.10.32:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
        server web2 10.10.10.33:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
    server web2 10.10.10.34:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
    server web2 10.10.10.35:88 cookie app1inst2 check inter 2000 rise 2 fall 5  
EOF  
cat >> /etc/init.d/haproxy <<EOF
#! /bin/sh  
set -e  
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/haproxy/sbin  
PROGDIR=/usr/local/haproxy  
PROGNAME=haproxy
DAEMON=\$PROGDIR/sbin/\$PROGNAME  
CONFIG=\$PROGDIR/\$PROGNAME.cfg  
PIDFILE=\$PROGDIR/\$PROGNAME.pid  
DESC="HAProxy daemon"
SCRIPTNAME=/etc/init.d/\$PROGNAME  
# Gracefully exit if the package has been removed.  
test -x \$DAEMON || exit 0  
start()  
{  
        echo -n "Starting \$DESC: \$PROGNAME"  
        \$DAEMON -f \$CONFIG  
        echo "."  
}  
stop()  
{  
        echo -n "Stopping \$DESC: \$PROGNAME"  
haproxy_pid=cat /usr/local/haproxy/haproxy.pid  
        kill \$haproxy_pid  
        echo "."  
}  
restart()  
{  
        echo -n "Restarting \$DESC: \$PROGNAME"  
        \$DAEMON -f \$CONFIG -p \$PIDFILE -sf \$(cat \$PIDFILE)  
        echo "."  
}  
case "\$1" in  
  start)  
        start  
        ;;  
  stop)  
        stop  
        ;;  
  restart)  
        restart  
        ;;  
  *)  
        echo "Usage: \$SCRIPTNAME {start|stop|restart}" >&2  
        exit 1  
        ;;  
esac   
exit 0  
EOF  
chmod +x /etc/rc.d/init.d/haproxy  
chkconfig --add haproxy  
chmod 777 /usr/local/haproxy/haproxy.pid  
sed -i '/SYSLOGD_OPTIONS/c\SYSLOGD_OPTIONS="-r -m 0"' /etc/sysconfig/syslog  
echo "local3.*        /var/log/haproxy.log" /etc/syslog.conf  
echo "local0.*        /var/log/haproxy.log" /etc/syslog.conf  
service syslog restart  
#启动haproxy  
# /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg  
#重启haproxy  
# /usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg -st `cat /usr/local/haproxy/haproxy.pid`   
#停止haproxy  
# killall haproxy  
# service haproxy start restart stop

haproxy是识别主机名的判断的主机名的判断的例子格式如下：

  
  acl url_aaa    hdr_dom(host)    www.aaa.com 
acl url_bbb    hdr_dom(host)    www.bbb.com 
acl tm_policy             hdr_dom(host) -i trade.gemini.taobao.net 
acl denali_policy         hdr_reg(host) -i ^(my.gemini.taobao.net|auction1.gemini.taobao.net)$ 
acl path_url163  path_beg  -i /163 
acl path_url_bbb path_beg  -i / 
use_backend aaa      if url_aaa 
use_backend bbb      if url_bbb 
use_backend url163   if url_aaa path_url163 
backend url163 
   mode http 
   balance roundrobin 
   option httpchk GET /163/test.jsp 
   server url163 10.10.10.31:8080 cookie 1 check inter 2000 rise 3 fall 3 maxconn 50000 
backend aaa 
   mode http 
   balance roundrobin 
   option httpchk GET /test.jsp 
   srever app_8080 10.10.10.32:8080 cookie 1 check inter 1500 rise 3 fall 3 maxconn 50000 
backend bbb 
   mode http 
   balance roundrobin 
   option httpchk GET /test.jsp 
   srever app_8080 10.10.10.33:8090 cookie 1 check inter 1500 rise 3 fall 3 maxconn 50000

haproxy端还要做lvs客户端模式，绑定回环口。

  
  #!/bin/bash  
SNS_VIP=10.10.10.88  
source /etc/rc.d/init.d/functions  
case "$1" in  
start)  
       ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP  
       /sbin/route add -host $SNS_VIP dev lo:0  
       echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore  
       echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce  
       echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore  
       echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce  
       echo "RealServer Start OK"  
       ;;  
stop)  
       ifconfig lo:0 down  
       route del $SNS_VIP >/dev/null 2>&1  
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore  
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce  
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore  
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce  
       echo "RealServer Stoped"  
       ;;  
*)  
       echo "Usage: $0 {start|stop}"  
       exit 1  
esac  
exit 0

下面是squid的设置

  
  #!/bin/bash  
wget  http://www.squid-cache.org/Versions/v2/2.6/squid-2.6.STABLE6.tar.bz2  
tar jxvf squid-2.6.STABLE6.tar.bz2  
./configure --prefix=/usr/local/squid \  
--enable-async-io=320 \  
--enable-storeio="aufs,diskd,ufs" \  
--enable-useragent-log \  
--enable-referer-log \  
--enable-kill-parent-hack \  
--enable-forward-log \  
--enable-snmp \  
--enable-cache-digests \  
--enable-default-err-language=Simplify_Chinese \  
--enable-epoll \  
--enable-removal-policies="heap,lru" \  
--enable-large-cache-files \  
--disable-internal-dns \  
--enable-x-accelerator-vary \  
--enable-follow-x-forwarded-for \  
--disable-ident-lookups \  
--with-large-files \  
--with-filedescriptors=65536
cat >> /usr/local/squid/etc/squid.conf <<EOF
visible_hostname cache1.taobao.com  
http_port 192.168.1.44:80 vhost vport  
icp_port 0  
cache_mem 512 MB  
cache_swap_low 90  
cache_swap_high 95  
maximum_object_size 20000 KB  
maximum_object_size_in_memory 4096 KB  
cache_dir ufs /tmp1 3000 32 256  
cache_store_log none  
emulate_httpd_log on  
efresh_pattern ^ftp:           1440    20%     10080  
refresh_pattern ^gopher:        1440    0%      1440  
refresh_pattern .               0       20%     4320  
negative_ttl 5 minutes  
positive_dns_ttl 6 hours  
negative_dns_ttl 1 minute  
connect_timeout 1 minute  
read_timeout 15 minutes  
request_timeout 5 minutes  
client_lifetime 1 day  
half_closed_clients on  
maximum_single_addr_tries 1  
uri_whitespace strip  
ie_refresh off  
logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh  
pid_filename /var/log/squid/squid.pid  
cache_log /var/log/squid/cache.log  
access_log /var/log/squid/access.log combined  
acl all src 0.0.0.0/0.0.0.0  
acl QUERY urlpath_regex cgi-bin .php .cgi .avi .wmv .rm .ram .mpg .mpeg .zip .exe  
cache deny QUERY  
acl picurl url_regex -i \.bmp$ \.png$ \.jpg$ \.gif$ \.jpeg$  
acl mystie1 referer_regex -i aaa  
http_access allow mystie1 picurl  
acl mystie2 referer_regex -i bbb  
http_access allow mystie2 picurl  
acl nullref referer_regex -i ^$  
http_access allow nullref  
acl hasref referer_regex -i .+  
http_access deny hasref picurl  
cache_peer 192.168.1.7 parent 80 0 no-query originserver no-digest name=all
cache_peer_domain all *.taobao.com  
cache_effective_user nobody  
cache_effective_group nobody  
acl localhost src 127.0.0.1  
acl my_other_proxy srcdomain .a.com  
follow_x_forwarded_for allow localhost  
follow_x_forwarded_for allow all   #允许转发 head ip 头  
acl_uses_indirect_client on     #只有2.6才有这这个个参数  
delay_pool_uses_indirect_client on  #只有2.6才有这这个个参数  
log_uses_indirect_client on    # 只有2.6才有这这个个参数  
#refresh_pattern ^ftp: 60 20% 10080  
#refresh_pattern ^gopher: 60 0% 1440  
#refresh_pattern ^gopher: 60 0% 1440  
#refresh_pattern . 0 20% 1440  

refresh_pattern -i \.js$        1440    50%     2880   reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private     
refresh_pattern -i \.html$      720     50%     1440    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private    
refresh_pattern -i \.jpg$       1440    90%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private    
refresh_pattern -i \.gif$       1440    90%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private    
refresh_pattern -i \.swf$       1440    90%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private    
refresh_pattern -i \.jpg$       1440    50%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private    
refresh_pattern -i \.png$       1440    50%     2880     reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private   
refresh_pattern -i \.bmp$       1440    50%     2880      reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private  
refresh_pattern -i \.doc$       1440    50%     2880       reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private  
refresh_pattern -i \.ppt$       1440    50%     2880      reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private   
refresh_pattern -i \.xls$       1440    50%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private     
refresh_pattern -i \.pdf$       1440    50%     2880   reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private      
refresh_pattern -i \.rar$       1440    50%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private     
refresh_pattern -i \.zip$       1440    50%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private     
refresh_pattern -i \.txt$       1440    50%     2880    reload-into-ims ignore-reload ignore-no-cache ignore-auth ignore-private
EOF  
#建立缓存和日志目录，并改变权限使squid能写入  
mkdir /tmp1  
mkdir /var/log/squid  
chown -R nobody:nobody /tmp1  
chmod 666 /tmp1  
chown -R nobody:nobody /var/log/squid  
#首次运行squid要先建立缓存  
/usr/local/squid/sbin/squid -z  
#启动squid  
echo "65535" > /proc/sys/fs/file-max  
ulimit -HSn 65535  
/usr/local/squid/sbin/squid

缓存的清理脚本是从洒哥那里搞到的

只是根据洒哥的脚本很简单的延伸了下，以前那个分享的脚本可以去除域名和特定的文件格式，然后我就想了能不能去除一个网址的所有jpg 或者是 www.92hezu.com/123/bbb/ 这样的。原来多家几个后缀，用grep过滤就ok了

qingli.sh www.xiuxiukan.com

qingli.sh jpg

qingli.sh xiuxiukan.com 123 bbb jpg

  
  #!/bin/sh 
squidcache_path="/squidcache"
squidclient_path="/home/local/squid/bin/squidclient"
#grep -a -r $1 $squidcache_path/* | grep "http:" | awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' > cache.txt 
if [[ "$1" == "swf" || "$1" == "png" || "$1" == "jpg" || "$1" == "ico" || "$1" == "gif" || "$1" == "css" || "$1" == "js" || "$1" == "html" || "$1" == "shtml" || "$1" == "htm"   ]]; then 
grep -a -r .$1 $squidcache_path/* | strings | grep "http:" | awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' | grep "$1$" | uniq > cache.txt 
else 
grep -a -r $1 $squidcache_path/* | strings | grep "http:" |grep $2$ |grep $3$|grep $4$|grep $5$ |grep $6$| awk -F 'http:' '{print "http:"$2;}' | awk -F\' '{print $1}' | uniq > cache.txt 
fi 
sed -i "s/\";$//g" cache.txt 
cat cache.txt | while read LINE 
do 
$squidclient_path -p 80 -m PURGE $LINE 
done