python爬漏洞库

一个简单的爬漏洞库，并整理到Excel中

# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import xlwt
import warnings
import sys
import time
reload(sys)
sys.setdefaultencoding('utf-8')

warnings.filterwarnings("ignore")

headers = {'user-agent':"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
cookies = {"csrftoken":"PpSmXuuzZqGEqQDel0MFFq9OPMorKKfD","sessionid":"208768211c8246235bc0019fadea1f85","left_menustatue_NSFOCUSRSAS":"3|0|https://192.168.30.9/template/index/"}

def get_policy_id():
    url1 = "获取漏洞编号"
    r = requests.get(url1,headers=headers,cookies=cookies,verify=False)
    html = r.text
    #print html

    ids = []
    soup = BeautifulSoup(html)
    for div_id in soup.find_all('div',class_='tree_node'):
        ids.append(div_id.attrs['id'].replace("_category",""))
    with open('policy_id.txt','w+') as f:
        for id in ids:
            for i in range(0,3):
                url2 = "获取漏洞名称"
                r2 = requests.get(url2,headers=headers,cookies=cookies,verify=False)
                html2 = r2.text
                soup2 = BeautifulSoup(html2)
                for policy_id in soup2.find_all('div',class_='tree_node'):
                    #print policy_id.attrs['id']
                    for vul in policy_id.find_all('a'):
                        vul_name = vul.string
                        f.write(id+":"+str(i)+":"+policy_id.attrs['id']+":"+vul.string+"\n")
        print "finish get_policy_id"

def get_policy_msg():
    ws = xlwt.Workbook(encoding = 'utf-8')
    sheet=ws.add_sheet(u'策略分类',cell_overwrite_ok=True)
    sheet.write(0,0,u"策略类别")
    sheet.write(0,1,u"等级")
    sheet.write(0,2,u"策略ID")
    sheet.write(0,3,u"漏洞名称")
    sheet.write(0,4,u"漏洞描述")
    sheet.write(0,5,u"解决方法")
    sheet.write(0,6,u"危险分值")
    sheet.write(0,7,u"危险插件")
    sheet.write(0,8,u"发现日期")
    sheet.write(0,9,u"CVE编号")
    sheet.write(0,10,u"CNNVD编号")
    sheet.write(0,11,u"CNCVE编号")
    sheet.write(0,12,u"BUGTRAQ")
    sheet.write(0,13,u"test2")
    sheet.write(0,14,u"CVSS评分")
    sheet.write(0,15,u"CNVD编号")
    now=time.time()
    now1=time.localtime(now)
    now2=time.strftime("%Y%m%d%H%M%S")
    vul_type = {'Vxworks':'40','Windows':'42','Solaris':'43','Linux':'44','BSD':'45',u'UNIX通用':'46',u'网络设备/防火墙':'47',u'系统无关':'48','HP-UX':'113','IRIX':'114','AIX':'115','Trub64 UNIX':'116','NetWare':'128',u'其他':'129','MacOX':'156'}
    try:
        with open("policy_id.txt","r") as f:
            lines =f.readlines()
            
            i = 1
            for line in lines:
                policy = line[:-1].split(":")
                
                if policy[2] != "no_data":
                    sheet.write(i,0,policy[0])
                    sheet.write(i,1,policy[1])
                    sheet.write(i,2,policy[2])
                    sheet.write(i,3,policy[3])
                    url = "https://192.168.30.9/获取漏洞详情"+policy[2]
                    s = requests.Session()
                    r = s.get(url,headers=headers,cookies=cookies,verify=False)
                    #print r.encoding
                    html = r.text
                    #print html
                    soup = BeautifulSoup(html)
                    for vul in soup.find_all("tr"):
                        for th in vul.find_all("th"):
                            name = th.string
                            pass

                            for td in vul.find_all("td"):
                                print td.string
                                if td.string is None:
                                    pass
                                elif name == u"漏洞描述":
                                    sheet.write(i,4,td.string)
                                elif name == u"解决办法":
                                    sheet.write(i,5,td.string)
                                elif name == u"危险分值":
                                    sheet.write(i,6,td.string)
                                elif name == u"危险插件":
                                    sheet.write(i,7,td.string)
                                elif name == u"发现日期":
                                    sheet.write(i,8,td.string)
                                elif name == u"CVE编号":
                                    sheet.write(i,9,td.string)
                                elif name == u"CNNVD编号":
                                    sheet.write(i,10,td.string)
                                elif name == u"CNCVE编号":
                                    sheet.write(i,11,td.string)
                                elif name == u"BUGTRAQ":
                                    sheet.write(i,12,td.string)
                                elif name == u"test2":
                                    sheet.write(i,13,td.string)
                                elif name == u"CVSS评分":
                                    sheet.write(i,14,td.string)
                                elif name == u"CNVD编号":
                                    sheet.write(i,15,td.string)
                                else:
                                    pass
                    
                    i += 1
                    print i
                    if i%100 == 0:
                        ws.save('%s.xls' % now2)

                else:
                    pass
    except:
        ws.save('%s.xls' % now2)
    ws.save('%s.xls' % now2)
if __name__ == "__main__":
    #get_policy_id()
    get_policy_msg()