requests库及在五个小例子中的应用

#python网络爬虫与信息提取
1.requests:自动爬取html页面自动网络请求提交
2.robot.txt：网络爬虫排除标准
3.projects实战

    3.1.小规模：（requess）爬取网页，玩转网页
    3.2.中规模：(scrapy)速度敏感 爬取网站，系列网站。
    3.3.大规模(定制开发):爬取全网

#requests入门
#通用的爬虫框架

import requests
url = "https://item.jd.com/100002749549.html"
try:
	r = requests.get(url)
	r.raise_for_status()
	r.encoding = r.apparent_encoding
	print(r.text[:1000])
except:
        print("爬取失败!")

#demo1.爬取亚马逊

import requests
r = requests.get("https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1")
print(r.status_code)
print(r.encoding)
r.encoding = r.apparent_encoding
print(r.encoding)
print(r.text)
print(r.request.headers)
kv = {'user-agent':'Mozilla/5.0'}
url = "https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1"
r = requests.get(url,headers = kv)
print(r.status_code)
print(r.request.headers)
print(r.text[:1000])

#d爬取亚马逊框架

import requests
url = "https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E" \
      "9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1"
try:
    kv = {'user-agent':'Mozilla/5.0'}
    r = requests.get(url,headers = kv)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text[1000:2000])
except:
    print("爬取失败！")
'''

#demo2.百度360搜索关键词提交
#2.1.百度

import requests
kv = {'wd':'python'}
r = requests.get("http://www.baidu.com/s",params = kv)
print(r.status_code)
print(r.request.url)
print(len(r.text))

#2.2 360

import requests
keyword = "python"
try:
   kv = {'q':keyword}
   r = requests.get("http://www.so.com/s",params = kv)
   print(r.request.url)
   r.raise_for_status()
   print(len(r.text))
except:
   print("爬取失败！")

#demo3.网络图片的爬取和存储
#格式：pictures.jpg

import requests
path = "E:\Pictures/xiaozhen.jpg"
url = "http://img0.dili360.com/ga/M00/00/7C/wKgBy1Q2a0OAMLwoAAizk5qejC0514.jpg@!rw9"
r= requests.get(url)
print(r.status_code)
with open(path,'wb') as f:
    f.write(r.content)
f.close()

#demo4.图片爬取全代码

import requests
import os
url = "https://imgsa.baidu.com/forum/w%3D580/sign=e731858242ed2e73fce98624b701a16d/6c6e78310a55b3193a8fe8734ba98226cefc17eb.jpg"
root = "E:\Pictures"
path = root + url.split('/')[-1]
try:
   if not os.path.exists(root):
       os.mkdir(root)
   if not os.path.exists(path):
       r = requests.get(url)
       with open(path,'wb') as f:
             f.write(r.content)
             f.close()
             print("文件保存成功")
   else:
        print("文件已经存在")
except:
   print("爬取失败")

#demo5.ip地址归属地查询

import requests
url = "http://www.ip138.com/ip.asp?ip="
r = requests.get(url+"202.204.80.112")
print(r.status_code)
print(r.text[-500:])

import requests
url = "http://m.ip138.com/ip.asp?ip="
try:
    r = requests.get(url + '202.204.80.112')
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    print(r.text[-500:])
except:
    print("爬取失败")