python dpkt SSL 流tcp payload（从三次握手开始到application data）和证书提取

 

# coding: utf-8
#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import print_function
import traceback
import argparse
import ipaddress
from binascii import hexlify
import socket
import struct
import sys
import dpkt
import textwrap
import os
from asn1crypto import x509
from dpkt import ssl, Packet
import pickle
from constants import PRETTY_NAMES
from cert_filter import CertFilter


global encrypted_streams
encrypted_streams = []  # change_cipher
global ssl_servers_certs
ssl_servers_certs = {}
global ssl_servers_with_client_hello
ssl_servers_with_client_hello = set()
global client_hello_set
client_hello_set = set()

global server_ip_set
server_ip_set = set()

global buffer
buffer = {}
need_more_parse = False


def tls_multi_factory_new(buf):
    """
    Attempt to parse one or more TLSRecord's out of buf
    :param buf: string containing SSL/TLS messages. May have an incomplete record on the end
    :return:  [TLSRecord] int, total bytes consumed, != len(buf) if an incomplete record was left at the end.
    Raises SSL3Exception.
    """
    i, n = 0, len(buf)
    msgs = []

    while i + 5 <= n:
        v = buf[i + 1:i + 3]
        if v in ssl.SSL3_VERSION_BYTES:
            try:
                msg = ssl.TLSRecord(buf[i:])
                msgs.append(msg)
            except dpkt.NeedData:
                break
        else:
            if i == 0:  ############################################ added
                raise ssl.SSL3Exception('Bad TLS version in buf: %r' % buf[i:i + 5])
            else:
                break

        i += len(msg)

    return msgs, i


class FlowDirection(object):
    OUT = 1
    IN = 2
    UNKNOWN = 3


class Extension(object):
    """
    Encapsulates TLS extensions.
    """

    def __init__(self, payload):
        self._type_id, payload = unpacker('H', payload)
        self._type_name = pretty_name('extension_type', self._type_id)
        self._length, payload = unpacker('H', payload)
        # Data contains an array with the 'raw' contents
        self._data = None
        # pretty_data contains an array with the 'beautified' contents
        self._pretty_data = None
        if self._length > 0:
            self._data, self._pretty_data = parse_extension(payload[:self._length],
                                                            self._type_name)

    def __str__(self):
        # Prints out data array in textual format
        return '{0}: {1}'.format(self._type_name, self._pretty_data)


class OP:
    CHECK_TLS_PACKET = 1
    MERGE_TLS_PACKET = 2


def analyze_packet(_timestamp, packet, nth, op):
    """
    Main analysis loop for pcap.
    """
    eth = dpkt.ethernet.Ethernet(packet)
    if isinstance(eth.data, dpkt.ip.IP):
        # print("timestamp:", _timestamp, "debug")
        parse_ip_packet(eth.data, nth, _timestamp, op)


def parse_arguments():
    """
    Parses command line arguments.
    """
    global filename
    global verboseprint
    global output_file
    global is_white_sample
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent('''\
Captures, parses and shows TLS Handshake packets
Copyright (C) 2015 Peter Mosmans [Go Forward]
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.'''))
    parser.add_argument('-r', '--read', metavar='FILE', action='store',
                        help='read from file (don\'t capture live packets)')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='increase output verbosity')
    parser.add_argument('-t', '--is_white_sample', action='store_true',
                        help='is white sample?')
    parser.add_argument('-o', '--output', action='store',
                        help='output file')
    args = parser.parse_args()

    if args.verbose:
        def verboseprint(*args):
            print('# ', end="")
            for arg in args:
                print(arg, end="")
            print()
    else:
        verboseprint = lambda *a: None
    if args.is_white_sample:
        print("OK, process white sample data.")
        is_white_sample = True
    else:
        print("OK, process black sample data.")
        is_white_sample = False
    filename = None
    if args.read:
        filename = args.read
    output_file = "demo_output.pickle"
    if args.output:
        output_file = args.output


def parse_ip_packet(ip, nth, timestamp, op):
    """
    Parses IP packet.
    """
    sys.stdout.flush()
    if isinstance(ip.data, dpkt.tcp.TCP):
        # print("****TCP packet found****", "tcp payload:", list(ip.data.data))
        """
        try:
            tls = dpkt.ssl.TLS(ip.data.data)
            if len(tls.records) < 1:
                return
        except Exception as e:
            print(e)
            return
        """
        parse_tcp_packet(ip, nth, timestamp, op)


# TLS  version
def check_tls_version(data):
    version2 = False
    version3 = False

    if len(data) > 2:
        # ssl
        tmp = struct.unpack("bbb", data[0:3])
    else:
        return version2, version3

    # SSL v2. OR Message body too short.
    if (tmp[0] & 0x80 == 0x80) and (((tmp[0] & 0x7f) << 8 | tmp[1]) > 9):
        version2 = True
    elif (tmp[1] != 3) or (tmp[2] > 3):  # 版本,SSL 3.0 or TLS 1.0, 1.1 and 1.2
        version3 = False
    elif (tmp[0] < 20) or (tmp[0] > 23):  # 类型错误
        pass
    else:
        version3 = True

    return version2, version3


def parse_tcp_packet(ip, nth, timestamp, op):
    """
    Parses TCP packet.
    """
    record_server_ip(ip)
    stream = ip.data.data
    if len(stream):
        record_data_flow(ip, stream, nth, timestamp)


# def record_server_ip(ip):
#     """
# >>> int(ipaddress.IPv4Address('192.168.0.1'))
# 3232235521
# >>> int(ipaddress.IPv4Address('0.0.0.1'))
# 1
# >>> int(ipaddress.IPv4Address('0.0.1.1'))
# 257
#     """
#     src_ip = socket.inet_ntoa(ip.src)
#     dst_ip = socket.inet_ntoa(ip.dst)
#     # 这里使用一个确定规则（小ip作为源），把相同方向的数据汇聚到一起
#     if int(ipaddress.IPv4Address(src_ip)) < int(ipaddress.IPv4Address(dst_ip)):
#         connection_key = "{}-{}".format(dst_ip, src_ip)
#         global server_ip_set
#         if connection_key not in server_ip_set:
#             server_ip_set.add(connection_key)
#             buffer[connection_key] = [{"out":[], "in":[]}]


def record_server_ip(ip):
    src_ip = '{0}:{1}'.format(socket.inet_ntoa(ip.src), ip.data.sport)
    dst_ip = '{0}:{1}'.format(socket.inet_ntoa(ip.dst), ip.data.dport)
    tcp = ip.data
    fin_flag = ( tcp.flags & dpkt.tcp.TH_FIN ) != 0
    syn_flag = ( tcp.flags & dpkt.tcp.TH_SYN ) != 0
    rst_flag = ( tcp.flags & dpkt.tcp.TH_RST ) != 0
    psh_flag = ( tcp.flags & dpkt.tcp.TH_PUSH) != 0
    ack_flag = ( tcp.flags & dpkt.tcp.TH_ACK ) != 0
    urg_flag = ( tcp.flags & dpkt.tcp.TH_URG ) != 0
    ece_flag = ( tcp.flags & dpkt.tcp.TH_ECE ) != 0
    cwr_flag = ( tcp.flags & dpkt.tcp.TH_CWR ) != 0