python爬虫

安装requests:

pip install requests

获取百度首页：

import requests
url = 'https://www.baidu.com/'
response = requests.get(url=url)
response.encoding = 'utf-8'
page_text = response.text
page_text
with open('baidu.html','w',encoding='utf-8') as fp:
    fp.write(page_text)
#print(type(page_text))
#print(response.encoding)

获取输入型返回页面：

import requests
url = 'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=62095104_19_oem_dg'
val = input('enter a key:')
params = {
    'wd':val
}
#解决UA伪装
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}
res = requests.get(url,params,headers=headers)
res.encoding = 'utf-8'
page = res.text
filename = val + '.html'
with open(filename,'w',encoding='utf-8') as fp:
    fp.write(page)

处理ajax动态加载请求，豆瓣爱情类电影：

import requests
url = 'https://movie.douban.com/j/chart/top_list'
start = input('从第几部开始：')
end = input('到第几部：')
#参数动态化
dic_2 = {
    'type': '13',
    'interval_id': '100:90',
    'action': '',
    'start': start,
    'limit': end
}
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}
res = requests.get(url,params=dic_2,headers=headers)
res.encoding = 'utf-8'
page_2 = res.json()
for dic in page_2:
    print(dic['title']+ ':' +dic['score'])