《Web安全之机器学习入门》笔记：第十三章 13.4 有向图识别僵尸网络

本小节通过有向图识别僵尸网络来示例网络安全领域的应用。

一、僵尸网络

上图为黑产控制僵尸网络发起攻击的示意图，通常来说黑产会对整个僵尸网络的僵尸主机发布相同的控制指令，所以通过统计一段时间内攻击源的ip地址以及被攻击者域名之间的关联关系，可以初步判断出哪些IP地址可能被同一黑产团体控制。

本小节中，僵尸网络的有向图处理如下所示：

二、数据集

文件位置：

filename="../data/etl-ip-domain-train.txt"

脱敏函数

with open(filename) as f:
    for line in f:
        line=line.strip('\n')
        ip,domain=line.split()

        ip=re.sub(r'\d$', '*', ip)
        domain= re.sub(r'\w{3}$', '*', domain)
        domain = re.sub(r'^\w{3}', '*', domain)

        print ("%s\t%s" % (ip,domain))

三、数据导入

逐行读入数据，按照攻击源ip地址建立hash表，hash表的键值key为被攻击者的域名，源码如下（note：书籍配套源码为python2环境，我的运行环境为python3，配套源码直接运行有问题，修改后如下所示）：

with open(filename) as f:
    for line in f:
        (ip,domain)=line.split("\t")
        if not ip=="0.0.0.0":
            if ip not in iplist:
                iplist[ip]={}

            iplist[ip][domain]=1

定义阈值R，攻击的域名超过R的IP地址才可以列入统计范围

for ip in iplist.keys():
    if len(iplist[ip]) >= R:
        goodiplist[ip]=1

定义Jarccard系数（交并比），作为衡量两个IP攻击集合相似度的方式，定义阈值N，党两个IP攻击的的域名jarccard大于等于N时才列入统计范围。

#jarccard系数
def get_len(d1,d2):
    ds1=set()
    for d in d1.keys():
        ds1.add(d)

    ds2=set()
    for d in d2.keys():
        ds2.add(d)
    return len(ds1&ds2)/len(ds1|ds2)

for ip1 in iplist.keys():
    for ip2 in iplist.keys():
        if not ip1 == ip2 :
            weight=get_len(iplist[ip1],iplist[ip2])
            if (weight >= N) and (ip1 in goodiplist.keys()) and (ip2 in goodiplist.keys()):
                #点不存在会自动添加
                G.add_edge(ip1,ip2,weight=weight)

四、查询分析

定义阈值M，当同一团伙的IP地址大于M后才显示结果

n_sub_graphs=nx.number_connected_components(G)
sub_graphs=nx.connected_component_subgraphs(G)


for i,sub_graph in enumerate(sub_graphs):
    n_nodes=len(sub_graph.nodes())
    if n_nodes >= M:
        print("Subgraph {0} has {1} nodes {2}".format(i,n_nodes,sub_graph.nodes()))

五、运行结果

配置阈值如下所示

#相似度
N=0.5
#黑客团伙IP最少个数
M=3
#黑客IP攻击目标最小个数
R=2

运行结果

Subgraph 0 has 546 nodes ['49.85.72.25*', '49.86.110.21*', '49.86.204.17*', '49.86.204.19*', '49.86.204.1*', '49.86.204.3*', '49.86.204.9*', '49.86.204.*', '49.86.213.14*', '49.86.213.1*', '49.86.213.17*', '49.86.213.2*', '49.86.214.22*', '49.86.72.8*', '58.16.204.4*', '58.18.138.15*', '58.19.62.*', '58.208.248.3*', '58.208.32.13*', '58.208.32.19*', '58.208.32.21*', '58.208.32.23*', '58.208.32.25*', '58.208.32.7*', '58.208.33.15*', '58.208.33.1*', '58.208.33.2*', '58.208.33.7*', '58.208.34.16*', '58.208.34.6*', '58.208.34.8*', '58.208.35.15*', '58.209.126.20*', '58.209.126.3*', '58.209.126.8*', '58.209.127.10*', '58.209.127.17*', '58.214.235.17*', '58.214.247.5*', '58.216.36.12*', '58.216.74.7*', '58.216.74.9*', '58.217.185.12*', '58.219.226.23*', '58.241.12.14*', '58.243.226.1*', '58.255.121.17*', '58.255.124.17*', '58.255.125.22*', '58.255.125.23*', '58.255.126.14*', '58.255.126.1*', '58.42.238.1*', '58.42.6.24*', '58.47.198.4*', '58.47.240.15*', '58.47.240.19*', '58.47.241.1*', '58.47.241.20*', '58.47.241.3*', '58.47.242.12*', '58.47.242.22*', '58.47.243.11*', '58.47.243.15*', '58.47.243.22*', '58.48.12.22*', '58.48.13.23*', '58.48.30.18*', '58.49.114.23*', '58.49.86.23*', '58.49.86.25*', '58.49.87.6*', '58.51.228.7*', '58.52.199.1*', '58.52.199.*', '58.52.58.15*', '58.52.58.21*', '58.54.249.23*', '58.59.203.20*', '58.60.4.9*', '58.61.49.1*', '58.63.109.25*', '59.172.136.22*', '59.172.137.10*', '59.172.160.10*', '59.172.206.23*', '59.173.161.16*', '59.174.254.10*', '59.174.254.20*', '59.174.254.2*', '59.174.254.25*', '59.174.255.*', '59.174.65.22*', '59.175.67.8*', '59.35.156.24*', '59.35.157.25*', '59.35.158.19*', '59.35.158.25*', '59.35.159.14*', '59.35.159.3*', '59.35.159.7*', '59.37.169.2*', '59.40.119.13*', '59.41.146.24*', '59.41.147.1*', '59.44.176.18*', '59.44.179.3*', '59.44.40.20*', '59.45.115.*', '59.46.184.22*', '59.47.7.11*', '59.47.7.12*', '59.53.175.2*', '59.53.67.20*', '59.55.100.13*', '59.55.100.18*', '59.55.100.2*', '59.55.100.23*', '59.55.100.8*', '59.55.101.22*', '59.55.101.24*', '59.55.101.3*', '59.55.101.6*', '59.55.101.7*', '59.55.102.13*', '59.55.102.15*', '59.55.102.24*', '59.55.102.3*', '59.55.102.7*', '59.55.103.20*', '59.55.103.2*', '59.55.103.3*', '59.55.103.5*', '59.55.103.6*', '59.55.103.8*', '59.55.121.21*', '59.55.121.3*', '59.55.122.23*', '59.55.122.6*', '59.55.147.11*', '59.55.147.13*', '59.55.147.18*', '59.55.147.20*', '59.55.147.3*', '59.55.147.7*', '59.55.147.9*', '59.55.148.11*', '59.55.148.18*', '59.55.158.11*', '59.55.158.23*', '59.55.158.4*', '59.55.158.9*', '59.55.159.13*', '59.55.159.2*', '59.55.159.9*', '59.55.58.10*', '59.55.58.11*', '59.55.58.12*', '59.55.58.13*', '59.55.58.14*', '59.55.58.1*', '59.55.58.15*', '59.55.58.17*', '59.55.58.18*', '59.55.58.19*', '59.55.58.22*', '59.55.58.2*', '59.55.58.3*', '59.55.58.4*', '59.55.58.5*', '59.55.59.10*', '59.55.59.11*', '59.55.59.12*', '59.55.59.13*', '59.55.59.14*', '59.55.59.15*', '59.55.59.17*', '59.55.59.18*', '59.55.59.21*', '59.55.59.2*', '59.55.59.4*', '59.55.59.*', '59.55.59.5*', '59.55.59.6*', '59.55.59.9*', '59.56.126.13*', '59.56.126.17*', '59.56.126.19*', '59.56.126.2*', '59.56.126.3*', '59.56.126.5*', '59.56.127.10*', '59.56.127.15*', '59.56.127.18*', '59.56.127.20*', '59.56.127.24*', '59.56.127.*', '59.56.133.15*', '59.56.180.2*', '59.56.44.22*', '59.56.44.2*', '59.56.44.*', '59.56.44.6*', '59.56.44.9*', '59.56.9.21*', '59.59.165.10*', '59.59.165.1*', '59.59.165.11*', '59.59.165.12*', '59.59.165.13*', '59.59.165.14*', '59.59.165.15*', '59.59.165.17*', '59.59.165.18*', '59.59.165.19*', '59.59.165.20*', '59.59.165.21*', '59.59.165.22*', '59.59.165.23*', '59.59.165.2*', '59.59.165.24*', '59.59.165.3*', '59.59.165.*', '59.59.165.4*', '59.59.165.6*', '59.59.165.7*', '59.59.165.8*', '59.59.165.9*', '59.60.126.5*', '59.61.134.11*', '59.61.134.14*', '59.62.28.*', '59.62.41.11*', '59.62.41.1*', '59.62.41.12*', '59.62.41.13*', '59.62.41.17*', '59.62.41.18*', '59.62.41.23*', '59.62.41.2*', '59.62.41.6*', '59.62.41.8*', '59.62.42.18*', '59.62.42.5*', '59.62.42.*', '59.62.42.7*', '59.62.42.8*', '59.63.248.4*', '59.63.28.17*', '59.63.28.20*', '59.63.58.*', '60.11.230.12*', '60.11.233.18*', '60.11.233.2*', '60.11.237.18*', '60.11.237.3*', '60.11.239.17*', '60.11.249.7*', '60.161.181.1*', '60.161.88.13*', '60.165.219.1*', '60.167.185.21*', '60.168.20.12*', '60.168.20.15*', '60.168.20.19*', '60.168.20.23*', '60.168.20.3*', '60.168.20.6*', '60.168.21.10*', '60.168.21.15*', '60.168.21.17*', '60.168.21.20*', '60.168.21.22*', '60.168.21.2*', '60.168.21.24*', '60.168.21.25*', '60.168.21.3*', '60.168.21.5*', '60.168.21.7*', '60.168.21.8*', '60.168.21.9*', '60.168.24.10*', '60.168.24.11*', '60.168.24.12*', '60.168.24.20*', '60.168.24.21*', '60.168.24.24*', '60.168.24.2*', '60.168.24.7*', '60.168.25.17*', '60.168.25.18*', '60.168.25.20*', '60.168.25.23*', '60.168.25.24*', '60.168.25.2*', '60.168.25.5*', '60.171.13.4*', '60.172.228.12*', '60.172.229.18*', '60.172.246.2*', '60.172.246.7*', '60.172.246.8*', '60.173.218.1*', '60.17.40.2*', '60.180.139.1*', '60.182.192.16*', '60.182.192.24*', '60.182.192.6*', '60.182.192.7*', '60.182.192.9*', '60.182.193.11*', '60.182.193.21*', '60.182.193.5*', '60.182.193.6*', '60.182.193.9*', '60.182.194.12*', '60.182.194.13*', '60.182.194.15*', '60.182.194.7*', '60.182.194.8*', '60.182.195.15*', '60.182.195.16*', '60.182.195.18*', '60.182.195.20*', '60.182.199.1*', '60.183.203.16*', '60.183.217.17*', '60.186.110.16*', '60.186.111.3*', '60.186.193.16*', '60.190.67.17*', '60.19.237.13*', '60.206.64.11*', '60.206.64.3*', '60.215.129.5*', '60.220.109.21*', '60.220.110.7*', '60.220.124.20*', '60.220.124.22*', '60.220.124.23*', '60.220.124.2*', '60.220.124.4*', '60.220.124.*', '60.220.124.6*', '60.220.124.7*', '60.220.124.8*', '60.220.124.9*', '60.220.125.10*', '60.220.125.13*', '60.220.125.23*', '60.220.125.24*', '60.220.125.6*', '60.220.128.15*', '60.220.128.16*', '60.220.130.16*', '60.220.130.18*', '60.220.130.19*', '60.220.130.2*', '60.220.130.3*', '60.220.130.8*', '60.220.131.12*', '60.220.131.14*', '60.220.131.15*', '60.220.131.16*', '60.220.131.22*', '60.220.131.2*', '60.220.131.5*', '60.220.132.13*', '60.220.179.3*', '60.220.206.13*', '60.220.206.18*', '60.220.218.2*', '60.220.224.20*', '60.220.55.11*', '60.220.57.*', '60.220.57.23*', '60.220.57.24*', '60.220.57.3*', '60.220.57.8*', '60.220.58.10*', '60.220.58.18*', '60.220.58.23*', '60.220.58.4*', '60.220.58.7*', '60.220.59.14*', '60.220.60.10*', '60.220.60.11*', '60.220.60.22*', '60.220.61.10*', '60.220.61.11*', '60.220.61.16*', '60.220.61.24*', '60.220.61.2*', '60.220.61.3*', '60.220.61.5*', '60.220.63.17*', '60.220.63.20*', '60.220.64.19*', '60.220.64.2*', '60.220.64.21*', '60.220.64.22*', '60.220.64.*', '60.220.65.20*', '60.220.65.22*', '60.220.65.4*', '60.220.65.7*', '60.220.65.8*', '60.220.65.*', '60.220.65.9*', '60.220.66.14*', '60.220.68.25*', '60.220.68.6*', '60.220.68.7*', '60.220.70.12*', '60.220.70.20*', '60.220.76.23*', '60.220.77.18*', '60.220.99.1*', '60.22.103.2*', '60.2.252.15*', '60.22.67.6*', '60.22.91.10*', '60.23.165.21*', '60.23.165.22*', '60.23.184.14*', '60.23.185.19*', '60.23.191.11*', '60.235.43.15*', '60.24.232.6*', '60.24.86.10*', '60.2.56.5*', '60.4.138.5*', '61.137.254.2*', '61.140.234.3*', '61.141.173.12*', '61.141.176.*', '61.143.16.10*', '61.143.16.13*', '61.143.16.23*', '61.143.16.6*', '61.143.17.1*', '61.143.17.18*', '61.143.17.19*', '61.143.17.5*', '61.143.18.16*', '61.143.18.20*', '61.143.18.4*', '61.143.19.18*', '61.143.19.6*', '61.143.19.7*', '61.143.19.9*', '61.143.20.12*', '61.143.21.8*', '61.143.22.1*', '61.143.22.20*', '61.143.22.3*', '61.143.22.8*', '61.143.23.19*', '61.143.23.20*', '61.143.23.2*', '61.143.23.*', '61.143.23.9*', '61.145.16.1*', '61.145.17.11*', '61.145.17.1*', '61.145.32.12*', '61.145.32.3*', '61.145.32.6*', '61.145.33.3*', '61.145.34.5*', '61.145.35.10*', '61.145.35.25*', '61.145.48.21*', '61.145.49.14*', '61.145.49.17*', '61.146.118.11*', '61.154.15.11*', '61.154.15.14*', '61.154.15.15*', '61.154.15.22*', '61.154.16.2*', '61.154.16.24*', '61.154.173.14*', '61.154.37.20*', '61.157.126.*', '61.157.210.11*', '61.157.215.13*', '61.157.215.20*', '61.157.218.10*', '61.157.218.16*', '61.157.218.21*', '61.157.218.22*', '61.157.218.2*', '61.157.218.5*', '61.157.218.9*', '61.157.219.12*', '61.157.219.13*', '61.157.219.18*', '61.157.219.20*', '61.157.219.21*', '61.157.219.2*', '61.157.219.4*', '61.157.219.6*', '61.161.28.16*', '61.164.59.12*', '61.164.65.24*', '61.174.123.23*', '61.174.50.23*', '61.178.81.3*', '61.180.116.22*', '61.183.212.2*', '61.185.87.4*', '61.187.202.11*', '61.187.202.23*', '61.188.148.12*', '61.188.148.4*', '61.188.148.7*', '61.188.148.*', '61.241.201.3*', '61.241.201.6*', '61.54.208.13*', '61.54.208.*', '61.54.211.3*', '61.54.3.24*', '58.16.28.12*', '58.16.49.16*', '58.255.125.16*', '59.63.249.1*', '58.255.125.14*', '58.255.124.20*', '58.255.125.18*', '58.255.126.19*', '58.255.126.25*', '58.255.127.7*', '58.255.127.9*', '59.174.255.23*', '61.153.108.13*', '61.185.119.11*', '60.175.122.4*', '61.134.46.19*', '61.161.28.8*', '58.255.127.11*', '59.37.23.18*', '61.140.234.24*', '59.63.28.21*', '61.185.119.12*']
Subgraph 1 has 10 nodes ['51.255.65.1*', '51.255.65.4*', '51.255.65.*', '51.255.65.7*', '51.255.65.9*', '51.255.65.8*', '51.255.65.5*', '51.255.65.3*', '51.255.65.6*', '51.255.65.2*']
Subgraph 5 has 4 nodes ['69.197.177.15*', '69.30.233.10*', '69.197.189.9*', '69.197.175.2*']
Subgraph 9 has 5 nodes ['91.200.12.13*', '91.200.12.1*', '91.200.12.*', '91.200.12.6*', '91.200.12.9*']

可视化