python爬虫的四种思路方法

普通的get请求
​以百度搜索为例子

'''
输入所要搜索的人物，返回搜索界面，将文件保存在文件中
'''
import urllib.parse
import urllib.request
#输入关键字
kw = input("请输入关键字：")
url = "https://www.baidu.com/s?"
#get参数
data = {
    'ie':'utf8',
    'wd':kw
}
#首先将字典转换为query_string格式
query_string = urllib.parse.urlencode(data)
url = url + query_string
#构建请求对象
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
#创建对象
request = urllib.request.Request(url=url,headers=headers)
resporse = urllib.request.urlopen(request)
#保存在文件中
filename = kw + '.html'
with open(filename,'wb') as f:
    f.write(resporse.read())

普通的​post请求
​以百度翻译为例子

'''
百度翻译，输入内容进行翻译
'''
import urllib.request
import urllib.parse

post_url = 'https://fanyi.baidu.com/sug'

form_data = {
    'kw':'like'
}
#构建请求对象
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=post_url,headers=headers)
#对post参数进行处理
formdata = urllib.parse.urlencode(form_data).encode('utf8')
response = urllib.request.urlopen(request,data=formdata)
print(response.read().decode('utf8'))

​普通ajax-get请求
​以豆瓣电影为例子

'''
豆瓣电影爬取
'''
import urllib.request
import urllib.parse

ajax_url = "https://movie.douban.com/j/chart/top_list?type=13&interval_id=100%3A90&action=&"
page = int(input("请输入要获取的页数："))
page2 = 20 * page
query_string = {
    'start':'0',
    'limit':page2
}
#拼接
query_string = urllib.parse.urlencode(query_string)
ajax_url = ajax_url + query_string
#构建请求对象
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=ajax_url,headers=headers)
response = urllib.request.urlopen(request)
with open("douban.txt",'wb') as f:
    f.write(response.read())

普通​ajax-post请求
​以肯德基为例子

'''
肯德基爬取
'''

import urllib.request
import urllib.parse
ajax_url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname'
cname = input("请输入城市:")
pageIndex = input('要查询的页码:')
pageSize = input('要查询的个数:')
formdata = {
    'cname': cname,
    'pid': '',
    'pageIndex': pageIndex,
    'pageSize': pageSize
}

#处理表单数据
formdata = urllib.parse.urlencode(formdata).encode('utf8')
#表头信息
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=ajax_url,headers=headers)
response = urllib.request.urlopen(request,data=formdata)
print(response.read().decode('utf8'))