python做数据统计_用python检查网络并做数据统计

一、启动一个tcp server，把送来的数据返回。

$ more echoserver.py

#!/usr/bin/env python

"""Simple server that listens on port 16000 and echos back every input to the client.

Connect to it with:

telnet localhost 16000

Terminate the connection by terminating telnet (typically Ctrl-] and then 'quit').

"""

from __future__ import print_function

from gevent.server import StreamServer

# this handler will be run for each incoming connection in a dedicated greenlet

def echo(socket, address):

print('New connection from %s:%s' % address)

socket.sendall(b'Welcome to the echo server! Type quit to exit.\r\n')

# using a makefile because we want to use readline()

rfileobj = socket.makefile(mode='rb')

while True:

line = rfileobj.readline()

if not line:

print("client disconnected")

break

if line.strip().lower() == b'quit':

print("client quit")

break

socket.sendall(line)

print(line)

rfileobj.close()

if __name__ == '__main__':

# to make the server use SSL, pass certfile and keyfile arguments to the constructor

server = StreamServer(('0.0.0.0', 16000), echo)

# to start the server asynchronously, use its start() method;

# we use blocking serve_forever() here because we have no other jobs

print('Starting echo server on port 16000')

server.serve_forever()

二、连接server，发送单个字符，并记录，100个做一次统计。

$ more tcptest.py

import subprocess, time, sys, errno

import socket

import numpy as np

def tcptest(host, port, interval=.1, count=None):

print host, port

s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)

s.connect((host, port))

i = 0

err = 0

t = []

while True:

try:

i += 1

t1 = time.time()

s.sendall(str(i)+'\n')

data = s.recv(1024)

t2 = time.time()

t.append(t2-t1)

if i % 100 == 0:

summary(t)

t = []

if count and i >= count:

break

time.sleep(interval)

except IOError as e:

print e

if e.errno == errno.EPIPE:

break

except Exception, e:

print e

err += 1

if err > 100:

break

summary(t)

s.close()

def summary(t):

if len(t) > 0:

print '%s %d packets(min/mean/median/max/std):%f/%f/%f/%f/%f' % (time.asctime(), len(t), np.min(t), np.mean(t),np.median(t)

, np.max(t), np.std(t))

sys.stdout.flush()

if __name__ == '__main__':

host, port = sys.argv[1], int(sys.argv[2])

tcptest(host, port)

------

日志类似于：

xxx.xxx.xxx.xx 16000

Fri Sep  2 16:59:26 2016 100 packets(min/mean/median/max/std):0.000338/0.000647/0.000624/0.002038/0.000269

Fri Sep  2 16:59:36 2016 100 packets(min/mean/median/max/std):0.000329/0.001173/0.000483/0.063784/0.006295

Fri Sep  2 16:59:46 2016 100 packets(min/mean/median/max/std):0.000332/0.000498/0.000477/0.000846/0.000102

Fri Sep  2 16:59:56 2016 100 packets(min/mean/median/max/std):0.000325/0.000546/0.000487/0.001736/0.000201

Fri Sep  2 17:00:06 2016 100 packets(min/mean/median/max/std):0.000371/0.000633/0.000576/0.005159/0.000474

Fri Sep  2 17:00:16 2016 100 packets(min/mean/median/max/std):0.000307/0.000663/0.000547/0.008853/0.000851

并在多个服务器启动这个进程。

三、从各个服务器获取日志

服务器记录在

$ more loghost.txt

msg0=172.41.17.127

msg1=172.41.17.128

msg2=172.41.17.129

msg3=172.41.17.130

msg4=172.41.17.131

msg5=172.41.17.132

msg6=172.41.17.135

msg7=172.41.17.136

循环从服务器获取日志：

$ more fetchlog.sh

#!/bin/bash

basedir=/tmp/ana

remotelogdir=/tmp

loghostfile=$basedir/loghost.txt

cat $loghostfile|while read line;do

dirname=`echo $line|awk -F = '{print $1}'`

ip=`echo $line|awk -F = '{print $2}'`

echo mkdir -p $basedir/$dirname;

mkdir -p $basedir/$dirname;

echo scp $ip:$remotelogdir/tcptest.log $basedir/$dirname;

scp $ip:$remotelogdir/tcptest.log $basedir/$dirname;

done

四、对日志文件做summary

$ more summary.sh

#!/bin/bash

basedir=/tmp/ana

datadir=$1

if [ "$datadir" == "" ];then

echo input datadir,pls.

exit 2;

esle

echo do summary datadir=$datadir

fi

cat /dev/null > $datadir/summary.log

#find dataline and date string

grep "min\/mean\/median\/max\/std" $datadir/tcptest.log |cut -c 1-10 |uniq|while read date;do

echo date="$date" ;

day=`echo $date|awk '{print $3}'`;

month=`echo $date|awk '{print $2}'`;

datefile=$month$day.log

echo split into datefile=$datefile;

echo "min/mean/median/max/std" > $datadir/$datefile ;

grep "$date" $datadir/tcptest.log |awk -F : '{print $4}' >> $datadir/$datefile;

echo $date >> $datadir/summary.log

python $basedir/summary.py $datadir/$datefile / >> $datadir/summary.log

done

--------

$ more summary.py

import sys, pandas;

if __name__ == '__main__':

file = sys.argv[1];

separator = sys.argv[2];

data = pandas.read_csv(file,sep = separator);

summary = data.describe();

print(summary.head(10));

print("end");

五、获取日志和分析全部过程串联在一起

]$ more doall.sh  #!/bin/bash basedir=/tmp/ana loghostfile=$basedir/loghost.txt $basedir/fetchlog.sh cat $loghostfile|while read line;do dirname=`echo $line|awk -F = '{print $1}'` $basedir/summary.sh $dirname done