爬虫python漏洞群_python3-爬取cnnvd漏洞信息

#!/usr/bin/env python3#-*- coding: utf-8 -*-#by 默不知然

importurllib.requestfrom urllib importparsefrom bs4 importBeautifulSoupimporthttp.cookiejarimportxlwtimportzlibimportreimporttimeimportxlsxwriterimportsysimportdatetimeimportpymysql'''运行方法：

python vulnerabilities_crawler 2017-10-01 2017-10-31 178

第一个为开始时间，第二个为结束时间，第三个为总页数。'''

#获得漏洞详情链接列表

defvulnerabilities_url_list(url,start_time,end_time):

header={'User-Agent': 'Mozilla/5.0 (Linux; Android 4.1.2; Nexus 7 Build/JZ054K) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19','Accept-Encoding': 'gzip, deflate','Referer': 'http://cnnvd.org.cn/web/vulnerability/queryLds.tag'}

data={'qstartdate':'2017-10-30', #---------------》开始日期

'qenddate':'2017-10-31' #---------------》结束日期

}

data['qstartdate'] =start_time

data['qenddate'] =end_time

data= parse.urlencode(data).encode('utf-8')

vulnerabilities_url_html= urllib.request.Request(url,headers=header,data=data)

vulnerabilities_url_cookie=http.cookiejar.CookieJar()

vulnerabilities_url_opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(vulnerabilities_url_cookie))

vulnerabilities_url_html=vulnerabilities_url_opener.open(vulnerabilities_url_html)

vulnerabilities_url_html= zlib.decompress(vulnerabilities_url_html.read(), 16+zlib.MAX_WBITS)

vulnerabilities_url_html=vulnerabilities_url_html.decode()#提取漏洞详情链接

response = r'href="(.+?)" target="_blank" class="a_title2"'vulnerabilities_link_list=re.compile(response).findall(vulnerabilities_url_html)#添加http前序

i =0for link invulnerabilities_link_list:

vulnerabilities_lists.append('http://cnnvd.org.cn'+vulnerabilities_link_list[i])

i+=1

print("已完成爬行第%d个漏洞链接"%i)

time.sleep(0.2)#漏洞信息爬取函数

defvulnerabilities_data(url):

header={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0','Accept-Encoding': 'gzip, deflate, sdch',

}

vulnerabilities_data_html= urllib.request.Request(url,headers=header)

vulnerabilities_data_cookie=http.cookiejar.CookieJar()

vulnerabilities_data_opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(vulnerabilities_data_cookie))

vulnerabilities_data_html=vulnerabilities_data_opener.open(vulnerabilities_data_html)

vulnerabilities_data_html= zlib.decompress(vulnerabilities_data_html.read(), 16+zlib.MAX_WBITS)

vulnerabilities_data_html=vulnerabilities_data_html.decode()globalvulnerabilities_result_list

vulnerabilities_result_list=[] #抓取信息列表命名

#添加漏洞信息详情

vulnerabilities_detainled_soup1 = BeautifulSoup(vulnerabilities_data_html,'html.parser')

vulnerabilities_detainled_data= vulnerabilities_detainled_soup1.find('div',attrs={'class':'detail_xq w770'})