普通的get请求
以百度搜索为例子
'''
输入所要搜索的人物,返回搜索界面,将文件保存在文件中
'''
import urllib.parse
import urllib.request
#输入关键字
kw = input("请输入关键字:")
url = "https://www.baidu.com/s?"
#get参数
data = {
'ie':'utf8',
'wd':kw
}
#首先将字典转换为query_string格式
query_string = urllib.parse.urlencode(data)
url = url + query_string
#构建请求对象
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
#创建对象
request = urllib.request.Request(url=url,headers=headers)
resporse = urllib.request.urlopen(request)
#保存在文件中
filename = kw + '.html'
with open(filename,'wb') as f:
f.write(resporse.read())
普通的post请求
以百度翻译为例子
'''
百度翻译,输入内容进行翻译
'''
import urllib.request
import urllib.parse
post_url = 'https://fanyi.baidu.com/sug'
form_data = {
'kw':'like'
}
#构建请求对象
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=post_url,headers=headers)
#对post参数进行处理
formdata = urllib.parse.urlencode(form_data).encode('utf8')
response = urllib.request.urlopen(request,data=formdata)
print(response.read().decode('utf8'))
普通ajax-get请求
以豆瓣电影为例子
'''
豆瓣电影爬取
'''
import urllib.request
import urllib.parse
ajax_url = "https://movie.douban.com/j/chart/top_list?type=13&interval_id=100%3A90&action=&"
page = int(input("请输入要获取的页数:"))
page2 = 20 * page
query_string = {
'start':'0',
'limit':page2
}
#拼接
query_string = urllib.parse.urlencode(query_string)
ajax_url = ajax_url + query_string
#构建请求对象
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=ajax_url,headers=headers)
response = urllib.request.urlopen(request)
with open("douban.txt",'wb') as f:
f.write(response.read())
普通ajax-post请求
以肯德基为例子
'''
肯德基爬取
'''
import urllib.request
import urllib.parse
ajax_url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname'
cname = input("请输入城市:")
pageIndex = input('要查询的页码:')
pageSize = input('要查询的个数:')
formdata = {
'cname': cname,
'pid': '',
'pageIndex': pageIndex,
'pageSize': pageSize
}
#处理表单数据
formdata = urllib.parse.urlencode(formdata).encode('utf8')
#表头信息
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
}
request = urllib.request.Request(url=ajax_url,headers=headers)
response = urllib.request.urlopen(request,data=formdata)
print(response.read().decode('utf8'))