Wooyun最新确认漏洞爬虫V0.02

Wooyun最新确认漏洞爬虫v0.02

#coding:utf-8
import requests
import re
from bs4 import BeautifulSoup
wooyun_confirm='http://www.wooyun.org/bugs/new_confirm'
wooyun_url='http://www.wooyun.org'
wooyun_url_result=re.compile('<a href="(/bugs/wooyun-\d*-\d*)">(.*?)</a>')
wooyun_data_result=re.compile(u'<h3>缺陷编号：.*?">(.*)</a>[\w\W]*漏洞标题：(.*?)</h3>[\s\S]*?<h3 class=\'wybug_corp\'>相关厂商：      <a href="http://www.wooyun.org/corps/(.*?)">[\w\W]*?<h3 class=\'wybug_author\'>漏洞作者：        <a href="http://www.wooyun.org/whitehats/(.*?)">[\w\W]*?<h3 class=\'wybug_date\'>提交时间：(.*?)</h3>[\w\W]*?<h3 class=\'wybug_type\'>漏洞类型：  (.*?)</h3>[\w\W]*?<p class="detail">漏洞Rank：(.*?) </p>')
url_page='http://www.wooyun.org/bugs/new_confirm/page/'
page_id=74#yeshu
req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
  'Accept':'text/html;q=0.9,*/*;q=0.8',
  'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  'Accept-Encoding':'gzip',
 'Connection':'close',
'Referer':None 
                          }


def get_html(url):
    return requests.get(url,headers=req_header).text



def spider_data(url):
    data_url=get_html(url)
    wooyun_data_list=wooyun_data_result.findall(data_url)
    for wooyun_bug_data in wooyun_data_list:
        print u"缺陷编号:%s\n漏洞标题:%s\n相关厂商:%s\n漏洞作者:%s\n提交时间:%s\n漏洞类型:%s\n漏洞Rank:%s\n\n\n"%(wooyun_bug_data[0],wooyun_bug_data[1].strip(),wooyun_bug_data[2].strip(),wooyun_bug_data[3].strip(),wooyun_bug_data[4].strip(),wooyun_bug_data[5].strip(),wooyun_bug_data[6].strip())

def get_data(html):
    wooyun_vul_list=wooyun_url_result.findall(html)
    for wooyun_vul_url in wooyun_vul_list:
        wooyun_url_page=wooyun_url+wooyun_vul_url[0]#每页每个漏洞的链接
        print "下一个次元之门"
        spider_data(wooyun_url_page)


def spider_all(url):
    for i in range(1,page_id):
        html=get_html(url+str(i))
        get_data(html)
        print i


if __name__ == '__main__':
    print '\033[0m'
    spider_all(url_page)

耗时:挺久
原因:正则.*?是匹配除\n以外的任意字符,因此在网页换行时会出现匹配不到的错误
网页上某些描述为”然后输出之后为’,并且’需要用\转义

Wooyun最新确认漏洞爬虫v0.01

#coding:utf-8
import requests
import re
from bs4 import BeautifulSoup
wooyun_confirm='http://www.wooyun.org/bugs/new_confirm'
wooyun_url='http://www.wooyun.org'
wooyun_url_result=re.compile('<a href="(/bugs/wooyun-\d*-\d*)">(.*?)</a>')
#wooyun_data_result=re.compile('<h3>.*?>(WooYun.*?)</a>.*?<h3 class="wybug_title">(.*?)</h3>.*?<h3 class="wybug_corp">.*?>(.*?)</a>.*?<h3 class="wybug_author">.*?>(.*?)</a>.*?<h3 class="wybug_type">(.*?)</h3>.*?<h3 class="wybug_level">(.*?)</h3>')
wooyun_data_result=re.compile('.*?class="wybug_title">(.*?)</h3>.*?class="wybug_corp"')
url_page='http://www.wooyun.org/bugs/new_confirm/page/'
page_id=74#yeshu
out_file=open('wooyun_data.txt','w')
#wooyun_name_result=re.compile('<a href=.*>(.*)</a>')
req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
  'Accept':'text/html;q=0.9,*/*;q=0.8',
  'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
  'Accept-Encoding':'gzip',
 'Connection':'close',
'Referer':None 
                          }


def get_html(url):
    return requests.get(url,headers=req_header).text



def spider_data(url):
    data_url=get_html(url)
    #wooyun_data_list=wooyun_data_result.findall(data_url)
    soup=BeautifulSoup(data_url)
    #wooyun_vul_id=soup.findall()
    wooyun_vul_title=soup.find('h3','wybug_title')
    #wooyun_vul_corp=soup.find_all()
    #wooyun_vul_author=
    wooyun_vul_type=soup.find('h3','wybug_type')
    #wooyun_vul_level=soup.find('h3','wybug_level')
    wooyun_vul_detail=soup.find_all('p','detail')
    #for wooyun_bug_data in wooyun_data_list:
        #print "%s\n"%wooyun_bug_data
        #print "1"
    print "%s\n"%wooyun_vul_title.string
    print "%s\n" %wooyun_vul_type.string
    for i in wooyun_vul_detail:
        print "%s\n"%i.string

def get_data(html):
    wooyun_vul_list=wooyun_url_result.findall(html)
    #wooyun_name_list=wooyun_name_result.findall(html)
    for wooyun_vul_url in wooyun_vul_list:
        wooyun_url_page=wooyun_url+wooyun_vul_url[0]#每页每个漏洞的链接
        print "下一个次元之门"
        spider_data(wooyun_url_page)
        print  "%s\n%s\n"%(wooyun_url_page,wooyun_vul_url[1])
        #out_file.write(wooyun_url_page,wooyun_vul_url[1])
        #print '\033[1;34;40m'
        #print vul_str

        #print '\033[0m'
        #out_file.write("%s\t%s"%(wooyun_url_page,wooyun_vul_url[1]))
        #print wooyun_vul_url


def spider_all(url):
    for i in range(1,page_id):
        html=get_html(url+str(i))
        get_data(html)
        #spider_data(html)
        print i


if __name__ == '__main__':
    print '\033[0m'
    #print '\033[1;34;40m'
    spider_all(url_page)
    #out_file.close()

耗时：挺久
代码是个半成品，很多问题，很多空格，很多很多。起初用正则，匹配的时候有点问题，怎么都匹配不到目标字符。后来采用BeautifulSoup，还是在细节匹配上有些问题，rank排序，完全没到那一步