用python检查网络并做数据统计

一、启动一个tcp server，把送来的数据返回。

$ more echoserver.py 
#!/usr/bin/env python
"""Simple server that listens on port 16000 and echos back every input to the client.


Connect to it with:
  telnet localhost 16000


Terminate the connection by terminating telnet (typically Ctrl-] and then 'quit').
"""
from __future__ import print_function
from gevent.server import StreamServer




# this handler will be run for each incoming connection in a dedicated greenlet
def echo(socket, address):
    print('New connection from %s:%s' % address)
    socket.sendall(b'Welcome to the echo server! Type quit to exit.\r\n')
    # using a makefile because we want to use readline()
    rfileobj = socket.makefile(mode='rb')
    while True:
        line = rfileobj.readline()
        if not line:
            print("client disconnected")
            break
        if line.strip().lower() == b'quit':
            print("client quit")
            break
        socket.sendall(line)
        print(line)
    rfileobj.close()


if __name__ == '__main__':
    # to make the server use SSL, pass certfile and keyfile arguments to the constructor
    server = StreamServer(('0.0.0.0', 16000), echo)
    # to start the server asynchronously, use its start() method;
    # we use blocking serve_forever() here because we have no other jobs
    print('Starting echo server on port 16000')
    server.serve_forever()

二、连接server，发送单个字符，并记录，100个做一次统计。

$ more tcptest.py 
import subprocess, time, sys, errno
import socket
import numpy as np


def tcptest(host, port, interval=.1, count=None):
  print host, port
  s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
  s.connect((host, port))
  i = 0
  err = 0
  t = []
  while True:
    try:
      i += 1
      t1 = time.time()
      s.sendall(str(i)+'\n') 
      data = s.recv(1024)
      t2 = time.time()
      t.append(t2-t1)
      if i % 100 == 0:
        summary(t)
        t = []
      if count and i >= count:
        break
      time.sleep(interval)
    except IOError as e:
      print e
      if e.errno == errno.EPIPE:
        break
    except Exception, e:
      print e
      err += 1
      if err > 100:
        break
  summary(t)
  s.close()


def summary(t):
  if len(t) > 0:
    print '%s %d packets(min/mean/median/max/std):%f/%f/%f/%f/%f' % (time.asctime(), len(t), np.min(t), np.mean(t),np.median(t)
, np.max(t), np.std(t))
    sys.stdout.flush()


if __name__ == '__main__':
  host, port = sys.argv[1], int(sys.argv[2])
  tcptest(host, port)

------

日志类似于：

xxx.xxx.xxx.xx 16000
Fri Sep  2 16:59:26 2016 100 packets(min/mean/median/max/std):0.000338/0.000647/0.000624/0.002038/0.000269
Fri Sep  2 16:59:36 2016 100 packets(min/mean/median/max/std):0.000329/0.001173/0.000483/0.063784/0.006295
Fri Sep  2 16:59:46 2016 100 packets(min/mean/median/max/std):0.000332/0.000498/0.000477/0.000846/0.000102
Fri Sep  2 16:59:56 2016 100 packets(min/mean/median/max/std):0.000325/0.000546/0.000487/0.001736/0.000201
Fri Sep  2 17:00:06 2016 100 packets(min/mean/median/max/std):0.000371/0.000633/0.000576/0.005159/0.000474
Fri Sep  2 17:00:16 2016 100 packets(min/mean/median/max/std):0.000307/0.000663/0.000547/0.008853/0.000851

并在多个服务器启动这个进程。

三、从各个服务器获取日志

服务器记录在

$ more loghost.txt 
msg0=172.41.17.127
msg1=172.41.17.128
msg2=172.41.17.129
msg3=172.41.17.130
msg4=172.41.17.131
msg5=172.41.17.132
msg6=172.41.17.135
msg7=172.41.17.136

循环从服务器获取日志：

$ more fetchlog.sh 
#!/bin/bash
basedir=/tmp/ana
remotelogdir=/tmp
loghostfile=$basedir/loghost.txt
cat $loghostfile|while read line;do
dirname=`echo $line|awk -F = '{print $1}'`
ip=`echo $line|awk -F = '{print $2}'`
echo mkdir -p $basedir/$dirname;
mkdir -p $basedir/$dirname;
echo scp $ip:$remotelogdir/tcptest.log $basedir/$dirname;
scp $ip:$remotelogdir/tcptest.log $basedir/$dirname;
done

四、对日志文件做summary

$ more summary.sh 
#!/bin/bash


basedir=/tmp/ana


datadir=$1
if [ "$datadir" == "" ];then
  echo input datadir,pls.
  exit 2;
esle
  echo do summary datadir=$datadir
fi


cat /dev/null > $datadir/summary.log


#find dataline and date string
grep "min\/mean\/median\/max\/std" $datadir/tcptest.log |cut -c 1-10 |uniq|while read date;do 
  echo date="$date" ;
  day=`echo $date|awk '{print $3}'`;
  month=`echo $date|awk '{print $2}'`;
  datefile=$month$day.log
  echo split into datefile=$datefile;
  echo "min/mean/median/max/std" > $datadir/$datefile ;
  grep "$date" $datadir/tcptest.log |awk -F : '{print $4}' >> $datadir/$datefile;
  echo $date >> $datadir/summary.log
  python $basedir/summary.py $datadir/$datefile / >> $datadir/summary.log
done

--------

$ more summary.py
import sys, pandas;


if __name__ == '__main__':
  file = sys.argv[1];
  separator = sys.argv[2];
  data = pandas.read_csv(file,sep = separator);
  summary = data.describe();
  print(summary.head(10));
  print("end");

五、获取日志和分析全部过程串联在一起

]$ more doall.sh 
#!/bin/bash
basedir=/tmp/ana
loghostfile=$basedir/loghost.txt


$basedir/fetchlog.sh


cat $loghostfile|while read line;do
dirname=`echo $line|awk -F = '{print $1}'`
$basedir/summary.sh $dirname
done