快速爬取网页
importurllib.request
url ='http://www.baidu.com'
response = urllib.request.urlopen(url)
将爬取的内容保存到本地
fhandle=open("C:/Work/python_work/python_spider_study/1.html","wb")
fhandle.write(content)
fhandle.close()
另外一种保存方法
filename = urllib.request.urlretrieve(url,filename="C:/Work/python_work/python_spider_study/51cto.html")
urllib.request.urlcleanup()
获取网页的状态码
response.getcode()
获取爬取的URL地址
response.geturl()
模拟浏览器访问网站
user_agent ='Mozilla/5.0 (Windows NT 10.0; WOW64; rv:57.0) Gecko/20100101 Firefox/57.0'
headers = { 'User-Agent': user_agent }
request = urllib.request.Request(url,headers= headers)
response = urllib.request.urlopen(request)
超时设置
response = urllib.request.urlopen(request,timeout=1)
URL地址中,搜索关键字是中文,则需要解决编码问题
keywd ='照片'
key_code = urllib.request.quote(keywd)
url = 'http://www.baidu.com/s?wd='+ key_code