import requests,time
from bs4 import BeautifulSoup
# for i in range(1,204):
# url = 'https://src.sjtu.edu.cn/rank/firm/0/?page=%s'%str(i)
# s=requests.get(url).text
# print(s)
# <td class="am-text-center">
# <a href="/list/firm/3086">山东省教育厅</a>
# </td>
def get_edu_name_data():
for i in range(1,204):
url = 'https://src.sjtu.edu.cn/rank/firm/0/?page=%s'%str(i) #最开始i是int类型,转换成str类型,填充到url中
#可以使用try进行容错处理
try:
s=requests.get(url).text
print('->正在获取第%s页面数据'%str(i))
soup = BeautifulSoup(s, 'lxml')
#这里使用bs解析,利用tr标签,tr标签中的class:row
edu1=soup.find_all('tr',attrs={'class': 'row'})
for edu in edu1:
edu_name=edu.a.string
print(edu_name)
#a+是进行后加
with open('eduname.txt','a+',encoding='utf-8') as f:
f.write(edu_name+'\n')
f.close()
except Exception as e:
time.sleep(1)
pass
#主函数调用
if __name__ == '__main__':
get_edu_name_data()
第一个是edu的爬取
import requests
import base64
#https://fofa.info/api/v1/search/all?#email=your_email&key=your_key&qbase64=dGl0bGU9ImJpbmci
def get_fofa_data(email,apikey):
for eduname in open('eduname.txt',encoding='utf-8'):
e=eduname.strip()
search='"%s" && country="CN" && title=="Error 404--Not Found"'%e
b=base64.b64encode(search.encode('utf-8'))
b=b.decode('utf-8')
url='https://fofa.info/api/v1/search/all?email=%s&key=%s&qbase64=%s'%(email,apikey,b)
s=requests.get(url).json()
print('查询->'+eduname)
print(url)
if s['size'] != 0:
print(eduname+'有数据啦!')
for ip in s['results']:
print(ip[0])
else:
print('没有数据')
if __name__ == '__main__':
email='xxx'
apikey='xxx'
get_fofa_data(email,apikey)
第二个是fofa