初识Python网络爬虫

1、安装requests库：pip install requests

2、初识网络爬虫命令

import requests
# 修改头部信息
# kv = {'user-agent':'Content-Type'}
## r = requests.get('http://www.moe.gov.cn',headers=kv)
# url = 'http://www.moe.gov.cn'
# r = requests.get(url,headers=kv)	
#
# 获取网页信息
r = requests.get("http://www.baidu.com")
# 状态码信息
print(r.status_code)
print(type(r))
print(r.headers)
# 返回内容
print(r.text)
# 页面编码信息
print(r.encoding)
# 将编码改为可执行的或可阅读的编码
print(r.apparent_encoding)
# 转换为可读编码
r.encoding = 'utf-8'
print(r.text)

3、爬取网页的通用代码框架

import requests
def getHTMLText(url):
    try:
        kv = {'user-agent':'Mozilla/5.0'}
        r = requests.get(url,timeout=30,headers=kv)
        r.raise_for_status() # 如果状态不是200，引发HTTPError异常
        r.encoding = r.apparent_encoding
        return r.text[-5000:]
    except:
        return "产生异常"
# 正常情况
if __name__=="__main__":
    url = "http://www.baidu.com"
    print(getHTMLText(url))
# 报错情况
if __name__=="__main__":
    url = "www.baidu.com"
    print(getHTMLText(url))

 4 HTTP协议

 

示例一：京东商品

import requests
url = "http://item.jd.com/2967929.html"
try:
    r = requests.get(url)
    r.raise_for_status
    r.encoding = r.apparent_encoding
    print(r.text[:1000])
except:
    print("爬取失败")

示例二：亚马逊商品

 初步代码：

import requests
r = requests.get("http://www.amazon.cn/gp/product/B01M8L5Z3Y")
print(r.status_code)
print(r.encoding)
r.encoding = r.apparent_encoding
print(r.encoding)
print(r.text)
# 查询头部信息
print(r.request.headers)

import requests
# 建立一个词典，设置请求的头部信息
kv = {'user-agent':'Mozilla/5.0'}
url = "http://www.amazon.cn/gp/product/B01M8L5Z3Y"
r = requests.get(url,headers=kv)
print(r.status_code)
print(r.request.headers)
print(r.text[:1000])

最终代码：

import requests
url = "http://www.amazon.cn/gp/product/B01M8L5Z3Y"
try:
    kv = {'user-agent':'Mozilla/5.0'}
    r = requests.get(url,headers=kv)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text[1000:2000])
except:
    print("爬取失败")

示例三：

1.百度关键字搜索

import requests
keyword = "Python"
try:
    kv = {'wd':keyword}
    r = requests.get("http://www.baidu.com/s",params=kv)
    print(r.request.url)
    r.raise_for_status()
    print(len(r.text))
except:  
    print("爬虫失败")

特别注意：r.request.url中没有s,切记！！！切记！！！ 

import requests
keyword = "水果"
def getHTMLText(url):
	try:
		kv = {'user-agent':'Mozilla/5.0'}
		kd = {'wd':keyword}
		r = requests.get(url,timeout=30,headers=kv,params=kd)
		r.raise_for_status()
		r.encoding = r.apparent_encoding
		return r.request.url
	except:
		return "产生异常"
if __name__=="__main__":
	url = "http://www.baidu.com/s"
	print(getHTMLText(url))

1  360关键字搜索

import requests
keyword = "Python"
try:
    kv = {'q':keyword}
    r = requests.get("http://www.so.com/s",params=kv)
    print(r.request.url)
    r.raise_for_status()
    print(len(r.text))
except:  
    print("爬虫失败")

示例四：图片爬取

import requests
import os
# 右击图片复制地址
url = "http://image.nationalgeographic.com.cn/2017/0211/20170211061910157.jpg"
root = "D://pics//"
path = root + url.split('/')[-1]
try:
    if not os.path.exists(root):    # 保存文件路径
        os.mkdir(root)
    if not os.path.exists(path):    # 写入文件
        r=requests.get(url)
        with open(path,'wb') as f:
            f.write(r.content)
            f.close()
            print("文件保存成功")
    else:
        print("文件已存在")
except:
    print("爬取失败")

示例五：IP地址查询

import requests
kv = {'user-agent':'Mozilla/5.0'}
url = "https://ipchaxun.com/39.107.222.100/"
r = requests.get(url,headers=kv)
print(r.status_code)
print(r.encoding)
print(r.text[-5000:-4000])