cve 爬虫_第一个爬虫

import requests

from lxmlimport etree

import re

import Sqloperate

sql = Sqloperate.Sql()

from bs4import BeautifulSoup

import xml.dom.minidom

from xml.dom.minidomimport parse

def all_scrapy():

url ="https://svn.freebsd.org/ports/head/security/vuxml/vuln.xml"

headers = {

"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0",

"Cookie":"__utma=1.1880838836.1607938851.1607938851.1607938851.1;__utmc=1;__utmz=1.1607938851.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",

"Host":"svn.freebsd.org",

"Upgrade-Insecure-Requests":"1",

"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

}

response = requests.get(url, headers=headers)

data = response.text

file0b = open('freebuf.xml','w',encoding='utf-8')

file0b.write(data)

file0b.close()

#得到XML文件

DOMTree = xml.dom.minidom.parse(r"freebuf.xml")

# 得到元素对象

collection = DOMTree.documentElement

#获得子标签

vulns = collection.getElementsByTagName("vuln")

#print(vulns)

#for i in range(len(vulns)):

for iin vulns:

#print(len(vulns))

#try:

package = i.getElementsByTagName("package")

cve = i.getElementsByTagName("cvename")

product = i.getElementsByTagName("name")

version = i.getElementsByTagName("lt")

#print(version)

#如果package不为空，cvename为空，执行下一个，

for pin package:

for cin cve:

if((p!=None and c==None)or(p==None and c!=None)):

continue

else:

for xin cve:

if(x==''):

continue

else:

cves = x.childNodes[0].data

for yin product:

if(y==''):

continue

else:

product1=y.childNodes[0].data

for zin version:

if(z==''):

continue

else:

version1 = z.childNodes[0].data

# print(version1)

os_type =""

vulntag ="Freebuf"

sql.insert_operate(cves, product1, version1, os_type, vulntag)

if __name__ =='__main__':

all_scrapy()