【Python】Python网络爬虫（一）

1 前言

本文列举了几个requests库的爬虫小程序，内容来自北京理工大学嵩天老师的视频教程，大家想入门学习爬虫的可以搜一下学习。点击进入Requests库官方教程。

2 Requests库使用实例

2.1 京东商品页面的爬取

import requests
# 通用代码框架
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status # 如果返回的不是200，则抛异常
        r.encoding = r.apparent_encoding # 将备选编码方式赋值给编码方式，功能是针对中文网页使用UTF-8编码
        return r.text[:500] # text内容可能过多，因此只显示前500字符
    except:
        return "产生异常"
if __name__ == "__main__":
    url = "https://item.jd.com/100010823452.html"
    print(getHTMLText(url))

2.2 亚马逊商品页面的爬取

import requests
# 通用代码框架
def getHTMLText(url):
    try:
        # 将user-agent从python-requests/2.22.0更改为Mozilla/5.0
        # 由于亚马逊网站有一定的反爬虫机制，上述更改可以绕过部分的机制，状态码会从503变为200 
        kv = {'user-agent':'Mozilla/5.0'}
        r = requests.get(url, headers = kv) 
        r.raise_for_status
        r.encoding = r.apparent_encoding
        return r.text[-500:]
    except:
        return "产生异常"
if __name__ == "__main__":
    url = "https://www.amazon.cn/dp/B00CWPIDHI?_encoding=UTF8&ref_=nav_kdpcwin"
    print(getHTMLText(url))

2.3 百度、360搜索关键词（百度目前这个api返回的是需要验证码，失效了）

import requests
# 通用代码框架
def getHTMLText(url):
    try:
        kv = {'wd':'Python'}
        r = requests.get(url, params = kv)
        r.raise_for_status
        r.encoding = r.apparent_encoding
        return r.text[-500:]
    except:
        return "产生异常"
if __name__ == "__main__":
    url = "http://www.baidu.com/s"
    print(getHTMLText(url))

2.4 网络图片的爬取与存储

import requests
import os
# 网页图片后缀名命名的图片，爬取下载到本地
def getHTMLText(url):
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):  
            r = requests.get(url)
            # 以二进制形式写入
            with open(path, 'wb') as f: 
                f.write(r.content)
                f.close
                print("图片保存成功")
        else:
            print("图片已经存在")
    except:
        return "产生异常"
if __name__ == "__main__":
    url = "http://image.ngchina.com.cn/2020/0221/20200221013711877.jpg"
    root = "C:/Users/Desktop/Python_爬虫/test/"
    path = root + url.split('/')[-1]
    print(getHTMLText(url))

2.5 IP地址归属地自动查询

import requests
def getHTMLText(url):
    try:
        kv = {'user-agent':'Mozilla/5.0'}
        r = requests.get(url + '55.55.55.55', headers = kv)
        r.raise_for_status
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"
if __name__ == "__main__":
    url = "http://www.ip138.com/ips138.asp?ip="
    print(getHTMLText(url))