邮件相关
发送邮件:
#!/usr/bin/python import sys; import smtplib; from email.MIMEText import MIMEText mail_host = sys.argv[1] mail_user = sys.argv[2] mail_pass = sys.argv[3] mail_from = sys.argv[4] mail_to = sys.argv[5] mail_to_list = mail_to.split(';') mail_subject = sys.argv[6] mail_content = sys.argv[7] msg = MIMEText(mail_content, _subtype = 'html', _charset = 'utf-8') msg['From'] = mail_from msg['Subject'] = mail_subject msg['To'] = mail_to msg['MIME-Version'] = "1.0" msg['Content-Type'] = "text/html" try: client = smtplib.SMTP() client.connect(mail_host) client.login(mail_user, mail_pass) client.sendmail(mail_from, mail_to_list, msg.as_string()) except: print "something error..."
用法:python ./send_mail.py $MAIL_HOST $MAIL_USER $MAIL_PASS $MAIL_FROM $MAIL_TO "$MAIL_SUBJECT" "$MAIL_CONTENT"
带附件版本:
#!/usr/bin/python import os import sys; import smtplib; from email.MIMEText import MIMEText from email.MIMEMultipart import MIMEMultipart mail_host = sys.argv[1] mail_user = sys.argv[2] mail_pass = sys.argv[3] mail_from = sys.argv[4] mail_to = sys.argv[5] mail_to_list = mail_to.split(';') mail_subject = sys.argv[6] mail_content = sys.argv[7] mail_attach = '' if len(sys.argv) > 8: mail_attach = sys.argv[8] #print mail_attach msg = MIMEMultipart() # body = MIMEText(mail_content, _subtype = 'html', _charset = 'utf-8') msg.attach(body) # if os.path.isfile(mail_attach): att = MIMEText(open(mail_attach,'rb').read(),'base64','gb2312') att["Content-Type"] = 'application/octet-stream' att["Content-Disposition"] = 'attachment;filename="' + os.path.basename(mail_attach) +'"' msg.attach(att) msg['From'] = mail_from msg['Subject'] = mail_subject msg['To'] = mail_to msg['MIME-Version'] = "1.0" msg['Content-Type'] = "text/html" try: client = smtplib.SMTP() client.connect(mail_host) client.login(mail_user, mail_pass) client.sendmail(mail_from, mail_to_list, msg.as_string()) except: print "something error..."
发送脚本:
#! /bin/bash export LC_ALL="en_US.UTF-8" if [ $# -lt 1 ] then echo "Usage: $0 process_name" exit -1 fi cd $(dirname $0) LOCALIP=`/sbin/ifconfig eth0 | grep "inet addr" | cut -f 2 -d ":" | cut -f 1 -d " "` HOSTNAME=`hostname` gday=`date +"%x %X"` MAIL_HOST="smtp.exmail.qq.com" MAIL_USER="op@test.com" MAIL_PASS="optest" MAIL_FROM="op@test.com" MAIL_TO="chenzhj@test.com" MAIL_SUBJECT="$HOSTNAME[$LOCALIP] $1 $gday" MAIL_CONTENT="" echo "$MAIL_SUBJECT" #echo "python ./send_mail.py" $MAIL_HOST $MAIL_USER $MAIL_PASS $MAIL_FROM $MAIL_TO "$MAIL_SUBJECT" "$MAIL_CONTENT" python ./send_mail.py $MAIL_HOST $MAIL_USER $MAIL_PASS $MAIL_FROM $MAIL_TO "$MAIL_SUBJECT" "$MAIL_CONTENT"
生成表格邮件:
#!/bin/sh TIME=`date +%Y-%m-%d -d "1 days ago"` echo "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">" echo "<html xmlns=\"http://www.w3.org/1999/xhtml\">" echo "<head>" echo "<meta http-equiv=\"Content-Type\" content=\"text/html;charset=gb2312\">" #echo "<meta http-equiv=Content-Type content=\"text/html; charset=utf-8\">" echo "</head>" echo " <body>" echo " <h2>今日半价mysql数据统计</h2>" echo ' <table border="1">' echo " <tr>" echo " <th>DateTime</th>" echo " <th>统计站点</th>" echo " <th>总商品数</th>" echo " <th>在架商品数</th>" echo " <th>下架商品数</th>" echo " <th>增量总数</th>" echo " <th>在架的增量数</th>" echo " </tr>" YESTERDAY=`date +%Y-%m-%d -d'-1 days'` cat ../data/static.$YESTERDAY | while read line do echo $line | awk ' { print "\t\t\t<tr>\n\t\t\t\t<td>'$TIME'</td>" for (i=1; i<=6; i++) { print "\t\t\t\t<td>"$i"</td>" } print "\t\t\t</tr>" }' done echo " </table>" echo " </body>" echo " </html>"
awk
1、传递外部shell参数(-v)、同时处理多文件(getline) 示例:
#!/bin/bash if [ $# -ne 2 ] then echo "Usage: $0 local_log bc_log" exit fi awk -v bc_log=$2 'BEGIN{ SRC=0; TAR=0; /*print "bc_log:",bc_log;*/ }{ SRC=$1; close(bc_log); while(getline TAR < bc_log ) { if(TAR>SRC) { print SRC; break; } if(TAR==SRC) { break; } } }
字典过滤
#! /bin/gawk -f BEGIN{ total=0; while( getline< "/home/hadoop/zhenjing/solr/2.data" >0) { dict[$1] = 1; total++; } #print total; } { if( $1 in dict ){ next } print $1 } END{ }
分析日志,计算平均时间
#! /bin/gawk -f START { sum_time["item.get"] = 0; sum_count["item.get"] = 0; } { tmp = substr($0, index($0, "method")); split(tmp, arr, "\""); method = arr[3]; if(method == "") { next; } tmp = substr($0, index($0, "time:")); split(tmp, arr, ":"); time = arr[2]; # print method, time; if(time > 0){ sum_time[method] = sum_time[method] + time; sum_count[method] = sum_count[method] + 1; } } END { for(method in sum_time){ if(sum_count[method] > 0){ print method " = " sum_time[method] / sum_count[method]; } } }
awk统计tomcat android机型:
#! /bin/gawk -f # tail /tmp/temai.txt | grep Android | cut -d" " -f 18-23 | cut -d')' -f1 BEGIN{ FS = "; "; } { if( $1 in dict_os ){ dict_os[$1] = dict_os[$1] + 1; } else { dict_os[$1] = 1; } if( $3 in dict_jixing ){ dict_jixing[$3] = dict_jixing[$3] + 1; } else { dict_jixing[$3] = 1; } } END { for(os in dict_os){ print dict_os[os] " " os; } for(jx in dict_jixing){ print dict_jixing[jx] " " jx;
awk平均分拆文件:
awk '{hash=int(NR % 10 ); print $0 >> "outfile_tmp_"hash }' weiboid.txt
时间相关
基于时间的循环:date命令
typeset readonly START_DATE=$(date --date="$1" +%Y%m%d);
for ((CUR_DATE=START_DATE; CUR_DATE<=END_DATE; CUR_DATE=$(date --date="${CUR_DATE} 1 days" +%Y%m%d) ))
Linux时钟分为系统时钟(System Clock)和硬件(Real Time Clock,简称RTC)时钟。系统时钟是指当前Linux Kernel中的时钟,而硬件时钟则是主板上由电池供电的时钟,这个硬件时钟可以在BIOS中进行设置。当Linux启动时,硬件时钟会去读取系统时钟的设置,然后系统时钟就会独立于硬件运作。Linux中的所有命令(包括函数)都是采用的系统时钟设置。在Linux中,用于时钟查看和设置的命令主要有date、hwclock和clock。其中,clock和hwclock用法相近。
date 010318242008.30(月日时分年.秒)
date 010318242008(月日时分年)
date -s 20080103
date -s 18:24
clock -w
hwclock --set --date="07/07/06 10:19" (月/日/年 时:分:秒)
硬件时钟与系统时钟同步:# hwclock --hctosys(hc代表硬件时间,sys代表系统时间)或者# clock --hctosys
系统时钟和硬件时钟同步:# hwclock --systohc或者# clock --systohc
xargs
echo "FILE.txt" | grep -v '^$' \
| xargs -L 1 -I {} find . -maxdepth 1 -type f -name "{}" \
| sort | xargs -L 1 basename | xargs md5sum
somecommand | xargs -s 50000 echo | xargs -I '{}' -s 100000 rm '{}'
LOCALIP=`/sbin/ifconfig eth1 | grep "inet addr" | cut -f 2 -d ":" | cut -f 1 -d " "`
LOCALIP="$(/sbin/ifconfig eth1 | sed -n '/inet addr/s/.*inet addr:\([0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*/\1/gp')"
LOCAL_IP="$(/sbin/ifconfig | sed -n 's/^.*inet addr\:\([[:digit:]\.]\{7,14\}\).*$/\1/gp' | head -1)";
array 相关
#! /bin/bash colors=(red green blue) element_count=${#colors[@]} # Special syntax to extract number of elements in array. # element_count=${#colors[*]} works also. # # The "@" variable allows word splitting within quotes #+ (extracts variables separated by whitespace). # # This corresponds to the behavior of "$@" and "$*" #+ in positional parameters. # array loop: # Way 1: index=0 while [ "$index" -lt "$element_count" ] do # List all the elements in the array. echo ${colors[$index]} # ${colors[index]} also works because it's within ${ ... } brackets. let "index = $index + 1" # Or: # ((index++)) done # Way 2: # Each array element listed on a separate line. # If this is not desired, use echo -n "${colors[$index]} " # Doing it with a "for" loop instead: for i in "${colors[@]}" do echo "$i" done echo echo "For test:" # Usage: ArrayCheckExist "$value" "${arr[@]}" function ArrayCheckExist { value=$1 i=0 for str in "$@" do ((i++)) if [ $i -eq 1 ]; then # first parameter is checked value continue fi if [[ "$value" == "$str" ]]; then #echo "$value" "$str" return 1 # exist fi done return 0 } dirs=() # empty array index=0 while read ip dir info tomcat port do if [[ $ip == "#" || -z $port ]] then continue fi # index=`echo -n $tomcat |md5sum|cut -d ' ' -f1` # get string md5sum ArrayCheckExist "$tomcat" "${dirs[@]}" if [ $? -eq 0 ]; then dirs[ ${#dirs[@]} ]=$tomcat fi done < ./di.conf for i in "${dirs[@]}" do echo "$i" done echo "Array function test:" ArrayCheckExist "${dirs[@]}" return_val=$? if [[ $return_val -gt 0 ]]; then echo "${dirs[0]} Exist." fi ArrayCheckExist "test" "${dirs[@]}" if [ $? -gt 0 ]; then echo "test Exist." fi ArrayCheckExist "test" if [ $? -gt 0 ]; then echo "test Exist." fi ArrayCheckExist "/usr/local/webserver/di/tomcat-di2" "${dirs[@]}" if [ $? -gt 0 ]; then echo "Exist." fi
统计脚本
solr:
cat shards | awk -F "QTime=" '{sum+=$2; cnt++; if($2> 500) slow++;} END{print cnt; print slow++; print sum/cnt; print slow/cnt}'
dc:
cat dc.log | grep -o "spending time:[0-9]\+" | awk -F: '{sum+= $2; cnt++; if($2 > 500) slow++ } END{print cnt; print slow; print sum/cnt; print slow/cnt}'
rts:
cat dc.log | grep -o "search rts used: [0-9]\+" | awk -F: '{sum+= $2; cnt++; if($2 > 20) slow++ } END{print cnt; print slow; print sum/cnt; print slow/cnt}'
searchserver:
cat searchserver.log.2013-07-08.log | grep -o "totalTime=[0-9]\+" | awk -F= '{sum+= $2; cnt++; if($2 > 500) slow++ } END{print cnt; print slow; print sum/cnt; print slow/cnt}'
appserver:
cat userTime-appserver-2013-07-25_* | awk '{sum+= $5; cnt++; if($5 > 500) slow++ } END{print cnt; print slow; print sum/cnt; print slow/cnt}'
solr相关
solr core数据对比
#! /bin/bash if [ $# -lt 3 ] then echo "Usage $0 query_file solr1 solr2 [out_file]" exit -1 fi OUT_FILE="/tmp/query" if [ $# -gt 3 ] then OUT_FILE=$4 fi while read query do query=`echo $query | sed 's/wt=javabin/wt=json/g'` # KEY: set wt=json for solr request. wget "http://$2/solr/C2C_TAOBAO_0/select/?$query" -q -O ${OUT_FILE}_0 wget "http://$3/solr/C2C_TAOBAO_0/select/?$query" -q -O ${OUT_FILE}_1 # del QTime from solr "json" response sed -i 's/"QTime":[0-9]\+,//g' ${OUT_FILE}_0 sed -i 's/"QTime":[0-9]\+,//g' ${OUT_FILE}_1 cmp ${OUT_FILE}_0 ${OUT_FILE}_1 > /dev/null if [ "$?" != 0 ] then echo "$query" fi done < $1
solr查询参数提取
#! /bin/gawk -f /select params=\{/,/\}/ { if( match($0, /\{[^\}]+\}/, str)){ pos = length(str[0]) - 2 print substr(str[0], 2, pos) } }
获取淘宝真实链接:
#! /bin/bash if [ $# -lt 1 ] then echo "Usage: $0 url" exit 1 fi export LC_ALL=en_US.UTF-8
OUT=log wget $1 -o $OUT -O tmp req_arr=`grep "Location:" $OUT | grep "s.click.taobao.com" | cut -d" " -f 2` taobao_req=`echo $req_arr | cut -d" " -f 2` refer_req=`echo $req_arr | cut -d" " -f 3` ext_key=`echo $refer_req | awk -F"et%3D" '{print $2}' | sed -e 's/%25/%/g'` #echo "$taobao_req&ref=&et=$ext_key" #echo "curl \"$taobao_req&ref=&et=$ext_key\" -I -e \"$refer_req\"" #curl -s "$taobao_req&ref=&et=$ext_key" -I -e "$refer_req" curl -s "$taobao_req&ref=&et=$ext_key" -I -e "$refer_req" | grep "^Location" | cut -d" " -f 2
日常数据备份脚本
#! /bin/bash export LC_ALL=en_US.UTF-8 echo `date` cd `dirname $0` pwd echo `date +%s` # back taday data DATE=`date +%Y-%m-%d` SQL_FILE="jira_$DATE.sql" /opt/lampp/bin/mysqldump --default-character-set=utf8 -h 10.1.22.40 -u jira -pkoudai123 jira > $SQL_FILE r1=`ls -ld $SQL_FILE | awk '{print int($5/1024)}'` DATA_FILE="jira_home_$DATE.tar.gz" tar czf $DATA_FILE /data/server/jira-home r2=`ls -ld $DATA_FILE | awk '{print int($5/1024)}'` echo $r1 echo $r2 echo `date +%s` if [[ $r1 -gt 1024 && $r2 -gt 1024 ]] then # del old data DELDAY=`date --date='-15 day' +%Y-%m-%d` echo "del data : $DELDAY" rm *$DELDAY* fi echo "`date +%s` FINISH !"
操作文件
#! /bin/bash if [ $# -lt 3 ] then echo "Usage $0 in_file out_file num" exit -1 fi INPUT=$1 OUTPUT=$2 num=100 if [ $# -gt 2 ] then num=$3 fi for i in `seq 0 $num` do echo "$i" awk -F ^ -v prefix=$i '{ if(NF > 10) print prefix$0}' ${INPUT} >> ${OUTPUT} done