requests模块
python中原生的一款基于网络请求的模块,功能强大,操作简单,效率高;
作用:模拟浏览器发送请求;
1. 指定url
2. 发起请求
3. 获取数据
4. 持久化存储
爬取百度首页源代码
import requests
if __name__ == "__main__":
url = 'https://www.baidu.com/'
response = requests.get(url=url)
page_text = response.text
with open('baidu.html', 'w', encoding='utf-8') as fp:
fp.write(page_text)
print('数据爬取结束')
UA:User-Agent(请求载体的身份标识)。
UA检测:门户网站的服务器会检测对应请求的载体身份标识,如果检测到请求的载体身份标识为某一款浏览器说明该请求是一个正常的请求。
但是,如果检测到请求的载体身份标识不是基于某一款浏览器的,则表示该请求为不正常的请求(爬虫),则服务器端就很有可能拒绝该次请求。
UA伪装:让爬虫对应的请求载体身份标识伪装成某一款浏览器。
百度关键字搜索
import requests
if __name__ == "__main__":
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
url = 'https://www.baidu.com/s?'
search = input('enter a word:')
params = {
'wd': search
}
response = requests.get(url=url, params=params, headers=headers)
page_text = response.text
fileName = search+'.html'
with open(fileName, 'w', encoding='utf-8') as fp:
fp.write(page_text)
print(fileName, '保存成功')
百度翻译
import requests
import json
if __name__ == "__main__":
post_url = 'https://fanyi.baidu.com/sug'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
word = input('enter a word:')
data = {
'kw': word
}
response = requests.post(url=post_url, data=data, headers=headers)
dic_obj = response.json()
fileName = word+'.json'
fp = open(fileName, 'w', encoding='utf-8')
json.dump(dic_obj, fp=fp, ensure_ascii=False)
print('爬取结束!!!')
豆瓣网
import requests
import json
if __name__ == "__main__":
url = 'https://movie.douban.com/j/chart/top_list'
param = {
'type': '24',
'interval_id': '100:90',
'action': '',
'start': '0',
'limit': '20',
}
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
response = requests.get(url=url, params=param, headers=headers)
list_data = response.json()
fp = open('./豆瓣.json', 'w', encoding='utf-8')
json.dump(list_data, fp=fp, ensure_ascii=False)
print('爬取结束')
肯德基店铺位置爬取
import requests
if __name__ == '__main__':
url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
place = input('输入要爬取的网页关键词:')
data = {
'cname': '',
'pid': '',
'keyword': place,
'pageIndex': '1',
'pageSize': '10',
}
response = requests.post(url=url, data=data, headers=headers)
page_text = response.text
fileName = place + '.txt'
with open(fileName, 'w', encoding='utf-8') as pf:
pf.write(page_text)
print(fileName, '保存ok')
- 药监总局
import requests
import json
if __name__ == "__main__":
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
id_list = []
all_data_list = []
url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
page=(int(input("请输入需要爬取的页数.")))
for page in range(1, 4):
page=str(page)
data = {
'on': 'true',
'page': page,
'pageSize': '15',
'productName': '',
'conditionType': '1',
'applyname': '',
'applysn': '',
}
json_ids = requests.post(url=url, headers=headers, data=data).json()
for dic in json_ids['list']:
id_list.append(dic['ID'])
post_url = "http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById"
for id in id_list:
data = {
'id': id,
}
detail_json = requests.post(url=post_url, headers=headers, data=data).json()
print(detail_json)
all_data_list.append(detail_json)
fp = open('./allData.json', 'w', encoding='utf-8')
json.dump(all_data_list, fp=fp, ensure_ascii=False)
print("爬取结束!!!")