python自动化挖掘、爬虫

import requests,time
from bs4 import BeautifulSoup
# for i in range(1,204):
#     url = 'https://src.sjtu.edu.cn/rank/firm/0/?page=%s'%str(i)
#     s=requests.get(url).text
#     print(s)


# <td class="am-text-center">
# <a href="/list/firm/3086">山东省教育厅</a>
# </td>

def get_edu_name_data():
    for i in range(1,204):
        url = 'https://src.sjtu.edu.cn/rank/firm/0/?page=%s'%str(i) #最开始i是int类型，转换成str类型，填充到url中
        #可以使用try进行容错处理
        try:
            s=requests.get(url).text
            print('->正在获取第%s页面数据'%str(i))
            soup = BeautifulSoup(s, 'lxml')
            #这里使用bs解析，利用tr标签，tr标签中的class：row
            edu1=soup.find_all('tr',attrs={'class': 'row'})
            for edu in edu1:
                edu_name=edu.a.string
                print(edu_name)
                #a+是进行后加
                with open('eduname.txt','a+',encoding='utf-8') as f:
                    f.write(edu_name+'\n')
                    f.close()
        
        except Exception as e:
            time.sleep(1)
            pass
#主函数调用
if __name__ == '__main__':
    get_edu_name_data()

第一个是edu的爬取

import requests
import base64

#https://fofa.info/api/v1/search/all?#email=your_email&key=your_key&qbase64=dGl0bGU9ImJpbmci



def get_fofa_data(email,apikey):
    for eduname in open('eduname.txt',encoding='utf-8'):
        e=eduname.strip()
        search='"%s" && country="CN" && title=="Error 404--Not Found"'%e
        b=base64.b64encode(search.encode('utf-8'))
        b=b.decode('utf-8')
        url='https://fofa.info/api/v1/search/all?email=%s&key=%s&qbase64=%s'%(email,apikey,b)
        s=requests.get(url).json()
        print('查询->'+eduname)
        print(url)
        if s['size'] != 0:
            print(eduname+'有数据啦！')
            for ip in s['results']:
                print(ip[0])
        else:
            print('没有数据')







if __name__ == '__main__':
    email='xxx'
    apikey='xxx'
    get_fofa_data(email,apikey)

第二个是fofa