爬虫requests库简单抓取页面信息功能实现（Python）

最新推荐文章于 2024-08-16 20:06:14 发布

LIJZ_Python

最新推荐文章于 2024-08-16 20:06:14 发布

阅读量1.2k

点赞数

分类专栏： Python 爬虫文章标签：爬虫 requests

本文链接：https://blog.csdn.net/yinhangxitong36/article/details/79641118

版权

Python 同时被 2 个专栏收录

74 篇文章 1 订阅

订阅专栏

爬虫

21 篇文章 0 订阅

订阅专栏

本文介绍了使用Python的requests库抓取网页信息的步骤，包括设置headers，随机选择User-Agent，发送GET请求，处理响应，并对获取的数据进行解析。示例代码展示了如何从猫眼电影网站抓取电影排行榜信息，包括电影名称、主演和上映时间，并将数据存储到本地txt文件。

摘要由CSDN通过智能技术生成

import requests
import re, json,time,random
from requests import RequestException


UserAgentList = [
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
    "Opera/8.0 (Windows NT 5.1; U; en)",
    "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 ",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER) ",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E) ",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
    "Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "UCWEB7.0.2.37/28/999",
    "NOKIA5700/ UCWEB7.0.2.37/28/999",
    "Openwave/ UCWEB7.0.2.37/28/999",
    "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999"
]


# get方式下载某一个页面
def get_one_page(url):
    # 构造headers,登录的时候要先手动登录一次，然后利用http的漏洞设置cookie。
    headers = {
        'User-Agent': random.choice(UserAgentList),
        "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
        'Accept': 'text/html,application/xhtml+xm...plication',
        'Connection': 'keep-alive',
        'host': 'www.renren.com',
        'Cookie': "anonymid=jf0jikul-viir8s; depovince=BJ; jebecookies=988ff23d-7901-4e57-824c-962ed1273e87|||||; _r01_=1; ick_login=3e279ee2-262a-48b3-bb3b-3d76e9eb3420; jebe_key=c19e2773-140d-4775-b0a9-70ce5a180170%7Ccfcd208495d565ef66e7dff9f98764da%7C1521603640931%7C0%7C1521603641662",
    }
    try:
        # 发送get请求，获取response响应
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            # 200 -> OK
            return response.text
        return None
    except RequestException:
        return None


# 把http://maoyan.com/board/4?offset=i*10
# 页面获取到
# 对top中一个页面的处理
def get_top(offset):
    # 入口url地址
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    # download one page
    html = get_one_page(url)
    # 写入到本地
    # 1. 提取信息
    for i in get_real_info(html):
        # 2. 存储到本地
        write_item(i)
        time.sleep(random.randint(1,3))


# 提取一个页面的信息
def get_real_info(html):
    reg = '''<p class="name">.*?title="([\s\S]*?)" [\s\S]*?<p class="star">
            ([\s\S]*?)</p>[\s\S]*?<p class="releasetime">([\s\S]*?)</p>'''
    pattern = re.compile(reg)
    results = re.findall(pattern, html)
    # 提取各个电影的信息以备保存
    for item in results:
        yield {
            'title': item[0].strip(' \n'),  # 名称
            'star': item[1].strip(' \n'),   # 主演
            'releasetime': item[2].strip(' \n'),  # 上映时间
        }

# 本地写到本地txt文件
def write_item(content):
    with open('maoyanTop100.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '\r\n')



if __name__ == '__main__':
    # 生成入口序列0， 10， 20， 30，。。。90
    for i in range(1):
        get_top(i*10)