在Ubuntu中使用pycap和dpkt进行抓包和解包

本意是做一个通过抓包来获取项目中接口数据，自动整理成接口文档的工具。
步骤：
1.在Ubuntu中安装好pypcap
sudo apt-get install python-pypcap
2.安装好dpkt包
pip install dpkt
或自行下载后安装https://pypi.python.org/pypi/dpkt/
3.代码中其它包自行安装

代码：

# -*- coding: utf-8 -*-
import pcap
import dpkt
import urllib
import socket
import fcntl
import sys
import json
import struct
reload(sys)
import re
sys.setdefaultencoding("utf-8")
from insert_data.rap2_db import RapDB

request_port = "8088"  # 过滤端口
sniffer = pcap.pcap(name="enp0s3")  # 监听的网卡名
sniffer.setfilter("tcp port %s" % request_port)  # 设置过滤
db = RapDB()


def get_ip_address(ifname):
    #  获取本地IP
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        0x8915,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15])
    )[20:24])


local_ip = get_ip_address('enp0s3')

request = []
response = []
http_request = ""
http_response = ""
requests = {}
requests2 = {}
responses = {}
responses2 = {}
for packet_time, packet_data in sniffer:
    # 循环获取到网络包
    eth = dpkt.ethernet.Ethernet(packet_data)
    ip = eth.data
    tcp = ip.data
    sipi = '%d.%d.%d.%d' % tuple(map(ord, list(ip.src)))
    sporti = tcp.sport
    dipi = '%d.%d.%d.%d' % tuple(map(ord, list(ip.dst)))
    dporti = tcp.dport
    ip_port = sipi + ':' + str(sporti) + '-' + dipi + ':' + str(dporti)

    if tcp.dport == int(request_port) and len(tcp.data) > 0:
        #  判断，非空包且目标端口匹配定义的端口才进来
        if ip_port not in requests:
            requests[ip_port] = []
        if ip_port not in requests2:
            requests2[ip_port] = tcp.data
        # print(requests2[ip_port])
        try:
            # 对抓到的包进行解包，如果报错，证明请求数据包不完整，继续等待下一次包进来，直到包完整
            http_request = dpkt.http.Request(requests2[ip_port])
            requests2[ip_port] = ''
            requests[ip_port].append(http_request)
        except dpkt.dpkt.NeedData:
            if tcp.data not in requests2[ip_port]:
                requests2[ip_port] += tcp.data
                try:
                    http_request = dpkt.http.Request(requests2[ip_port])
                    requests2[ip_port] = ''
                    requests[ip_port].append(http_request)
                except:
                    pass
        except dpkt.dpkt.UnpackError:
            if tcp.data not in requests2[ip_port]:
                requests2[ip_port] += tcp.data
                try:
                    http_request = dpkt.http.Request(requests2[ip_port])
                    requests2[ip_port] = ''
                    requests[ip_port].append(http_request)
                except:
                    pass
    elif tcp.sport == int(request_port) and len(tcp.data) > 0:
        # 判断，如果请求端口端口匹配定义的端口，且包不为空才进来
        if ip_port not in responses:
           responses[ip_port] = []
        if ip_port not in responses2:
            responses2[ip_port] = tcp.data
        try:
            http_response = dpkt.http.Response(responses2[ip_port])
            responses2[ip_port] = ''
            responses[ip_port].append(http_response)
            # print(http_response.body)
        except dpkt.dpkt.UnpackError:
            if tcp.data not in responses2[ip_port]:
                responses2[ip_port] += tcp.data
                try:
                    http_response = dpkt.http.Response(responses2[ip_port])
                    responses2[ip_port] = ''
                    responses[ip_port].append(http_response)
                except:
                    pass
    ip_port_back = ip_port.split("-")[1] + "-" + ip_port.split("-")[0]
    # print "imin"
    if ip_port in responses and ip_port_back in requests:
        if responses[ip_port] and requests[ip_port_back]:
            if len(responses[ip_port]) == len(requests[ip_port_back]):
                # 在一次TCP连接中可能发起多个请求，这里如果请求数和返回数一致，则判断该次TCP连接已完成，进入到数据分析阶段
                for i in range(len(responses[ip_port])):
                    try:
                        json_data = re.compile('^{\n* *"(.*?)":.*?}$', re.S).findall(responses[ip_port][i].body)
                        #  匹配返回数据，如果是json格式才要，不然判断为非接口。丢弃此次数据
                    except TypeError:
                        json_data = ""
                    # print json_data
                    if json_data:  #

                        # print(len(responses[ip_port]), len(requests[ip_port]))
                        print("\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
                        print "http_request_url:", requests[ip_port_back][i].uri
                        print "http_request_method:", requests[ip_port_back][i].method
                        print "http_request_body:", requests[ip_port_back][i].body
                        print "http_request_header:", requests[ip_port_back][i].headers

                        print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")
                        print "http_response_status:", responses[ip_port][i].status
                        print "http_response_body:", responses[ip_port][i].body
                        print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^")

                        #  以下是对数据做存储，不通用,可根据自己需求自己处理
                        http_response = ""
                        http_request = ""
                        url = requests[ip_port_back][i].uri
                        method = requests[ip_port_back][i].method
                        request_body = requests[ip_port_back][i].body
                        response_body = responses[ip_port][i].body
                        response_status = responses[ip_port][i].status
                        model_name = url.split("/")[1]

                        if int(response_status) == 200:
                            model_num = db.model_insert(model_name)
                            if ".do" not in url:
                                interface_name = url.split("/")[-1].split("?")[0]
                            else:
                                interface_name = url.split("/")[-1].split(".")[0]
                            print "model_name:", model_name
                            print "interface_name:", interface_name

                            param_dics = {}
                            # if ".do" not in url:
                            #     continue
                            if "?" in url:
                                interface_url = url.split("?")[0]
                                interface_id = db.interface_insert(interface_name, interface_url, method, "", model_num)
                                params_url = url.split("?")[1]
                                if "&" in params_url:

                                    params = params_url.split("&")
                                else:
                                    params = [params_url]
                                # print "params:", params
                                for param in params:
                                    param_dic = param.split("=")
                                    param_dics[param_dic[0]] = param_dic[1]

                                print "param_dics:", param_dics
                            else:
                                interface_id = db.interface_insert(interface_name, url, method, "", model_num)
                            if param_dics:
                                param_dics = json.dumps(param_dics)
                                db.properties_insert("request", param_dics, interface_id, model_num, "url参数")
                            if request_body != "":
                                try:
                                    json.loads(request_body)
                                    param_type = ";Json格式"
                                except ValueError:
                                    request_bodys = {}
                                    if "&" in request_body:
                                        bodys = request_body.split("&")
                                    else:
                                        bodys = [request_body]
                                    for body in bodys:
                                        key = body.split("=")[0]
                                        value = body.split("=")[1]
                                        request_bodys[key] = value
                                    request_body = json.dumps(request_bodys)
                                    param_type = ";表单格式"
                                db.properties_insert("request", request_body, interface_id, model_num, "request参数"+param_type)
                            db.properties_insert("response", response_body, interface_id, model_num, "response参数")

                        param_dics = ""
                        interface_name = ""
                        interface_id = ""
                        model_name = ""
                        model_num = ""
                        requests.pop(ip_port_back)
                        responses.pop(ip_port)

                        print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"

                    else:
                        print "Data is captured, but not json data !"
                        http_response = ""
                        http_request = ""
                        requests.pop(ip_port_back)
                        responses.pop(ip_port)
                        continue

                # responses.clear()
                # print responses
                # requests.clear()
                # print requests