Python分布式爬虫实战 - 豆瓣读书

本实例从零到一实现豆瓣读书的所有标签的分布式爬虫编写

本实例使用到的工具:

• IDE:Pycharm
• 工具:Python,Scrapy,linux,mysql,redis
• 需要用到的模块:scrapy pymysql scrapy_redis selenium
• 抓取内容:书名,作者,出版日期,价格,评分,参与评分人数,评论数量,书籍类型

先来捋捋思路:

• step1.爬取所有标签页面的链接,保存到数据库
• step2.爬取每个标签所有内容页的链接
• step3.分布式爬取每个内容页(重点)
• step4:linux运行scrapy爬虫

废话不多说,直接开搞

1.爬取所有标签页面的链接,保存到数据库

• 这里为了方便,使用requests库进行爬取

import requests
from lxml import etree

# UA,不必多说了吧
header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
}
def crawl_tag_links(url):
    # 爬取总标签页面,也就是这个 "https://book.douban.com/tag/?view=cloud"
    response = requests.get(url, headers=header)
    e = etree.HTML(response.text)
    # 取下所有标签的链接(我这边一共120个标签URL)
    tag_links = e.xpath("//table[@class='tagCol']//a/@href")
    # 取下来的链接是网址的后部分,比如[/小说,/历史......],所以需要补全网址
    tag_links = [f"https://book.douban.com{i}" for i in tag_links]

• 保存到mysql数据库

import pymysql

def save_tag_links(links):
    # 建立数据库对象 注意修改数据库ip地址和账号密码
    conn = pymysql.connect("192.168.2.208", "root", "123456", "douban")
    # 游标对象
    cursor = conn.cursor()
    # 查询数据表是否存在
    # 返回1表示存在 0表示不存在
    if not cursor.execute("show tables like 'tag_links'"):
        # 创建数据表,这里命名为tag_links
        cursor.execute(
            """
            create table tag_links(
            id int primary key auto_increment,
            url varchar(100),
            status int
            )
            """
        )
    # 准备sql语句
    sql = "insert into tag_links values (%s,%s,%s)"
    # 准备插入数据库的数据
    # 第一个0是数据库的id列,插入数据时候id这一字段是自增的,所以给个0它就可以了
    # 第二个link就是每个标签页的url
    # 第三个0 表示还没被爬取,之后爬取这个标签页面的时候爬取成功后修改这里的0为1
    #               表示已经爬取过..这样哪怕发生意外也不用从新爬取了
    insert_links = [(0, link, 0) for link in links]
    try:
        # 批量插入数据
        cursor.executemany(sql, insert_links)
        # 注意插入数据是事务操作,需要提交
        conn.commit()
    except Exception as err:
        # 出现错误,回滚操作
        conn.rollback()
        print(err)
    finally:
        cursor.close()
        conn.close()

• 完整代码(写成一个类方便以后调用)

import requests, pymysql
from lxml import etree


class TagSpider():
    def __init__(self):
        # UA
        self.header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
        }

    def crawl_tag_links(self, url):
        # 爬取所有热门标签页面
        response = requests.get(url, headers=self.header)
        e = etree.HTML(response.text)
        # 取下所有标签的链接
        tag_links = e.xpath("//table[@class='tagCol']//a/@href")
        # 取下来的链接是网址的后部分,比如 (/小说),所以需要补全网址
        tag_links = [f"https://book.douban.com{i}" for i in tag_links]
        # 保存链接到mysql数据库
        self.save_tag_links(tag_links)

    def save_tag_links(self, links):
        # 建立数据库对象
        conn = pymysql.connect("192.168.2.208", "root", "123456", "douban")
        # 游标对象
        cursor = conn.cursor()
        # 查询数据表是否存在
        # 返回1表示存在 0表示不存在
        if not cursor.execute("show tables like 'tag_links'"):
            # 创建数据表,这里命名为tag_links
            cursor.execute(
                """
                create table tag_links(
                id int primary key auto_increment,
                url varchar(100),
                status int
                )
                """
            )
        # 准备sql语句
        sql = "insert into tag_links values (%s,%s,%s)"
        # 准备插入数据库的数据
        # 第一个0是数据库的id列,插入数据时候id这一字段是自增的,所以给个0它就可以了
        # 第二个link就是每个标签页的url
        # 第三个0 表示还没被爬取,之后爬取这个标签页面的时候爬取成功后修改这里的0为1
        #               表示已经爬取过..这样哪怕发生意外也不用从新爬取了
        insert_links = [(0, link, 0) for link in links]
        try:
            # 批量插入数据
            cursor.executemany(sql, insert_links)
            # 注意插入数据是事务操作,需要提交
            conn.commit()
        except Exception as err:
            # 出现错误,回滚操作
            conn.rollback()
            print(err)
        finally:
            cursor.close()
            conn.close()


if __name__ == '__main__':
    # 所有热门标签页的URL
    url = "https://book.douban.com/tag/?view=cloud"
    # 创建对象实例
    get_tag_links = TagSpider()
    # 开始爬取所有标签
    get_tag_links.crawl_tag_links(url)

step2.爬取每个标签所有内容页的链接

Ps:貌似豆瓣有限制,只能查看每个标签的前50页
大致估算了一下,120(标签) x 50(页) x20(个内容页) = 12w条数据
为了节省更多的爬取时间,下面开始使用scrapy爬取

• settings.py 配置文件

BOT_NAME = 'doubandushulinks'
SPIDER_MODULES = ['doubandushulinks.spiders']
NEWSPIDER_MODULE = 'doubandushulinks.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
   'doubandushulinks.pipelines.DoubandushulinksPipeline': 300,
}

• 爬虫文件

思路:分别爬取每个标签前50页,如遇上"没有找到符合条件的图书"就跳过…

# -*- coding: utf-8 -*-
import scrapy, pymysql, re
from urllib.parse import unquote


class DoubanlinksSpider(scrapy.Spider):
    name = 'doubanlinks'
    allowed_domains = ['douban.com']
    # 数据库对象
    conn = pymysql.connect("192.168.2.208", "root", "123456", "douban")
    # 游标
    cursor = conn.cursor()
    # 数据库中提取status为0(表示没爬取过)的URL
    cursor.execute("select url from tag_links where status = 0")
    urls = cursor.fetchall()
    # 养成良好的习惯,用完记得关闭数据库对象
    cursor.close()
    conn.close()
    # 每个标签url后补上页面数,每页编号相隔20
    start_urls = [f"{url[0]}?start={j}" for url in urls for j in range(0, 1000, 20)]

    def parse(self, response):
        # 如果页面出现 "没有找到符合条件的图书" 表示已经到达50也以后了
        if response.xpath("//p[@class='pl2']/text()").extract_first != "没有找到符合条件的图书":
            # 每个链接对应有标签名,保存下来日后做数据分析用
            tag = unquote(re.findall(r"tag/(.+)\?.+", response.url)[0])
            # 提取每一页的所有内容页链接
            content_links = response.xpath("//h2/a/@href").extract()
            # 准备存入数据库
            # 前面的0对应id,最后的0表示没有被爬取过,作用在之前解析过了
            item = {"data": [(0, url, tag, 0) for url in content_links]}
            # 爬取下来的数据只要轻轻的yield一下就可以交给管道处理了
            yield item

• pipelines.py 管道

就像我们吃东西进肚子里所经过的大肠小肠十二指肠…

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql

class DoubandushulinksPipeline(object):
    # 开始scrapy时调用以下函数
    def open_spider(self,spider):
        # 创建数据库对象
        self.conn = pymysql.connect("192.168.2.208","root","123456","douban")
        self.cursor = self.conn.cursor()
        # 如果没有content_links这个表就创建一个
        if not self.cursor.execute("show tables like 'content_links'"):
            self.cursor.execute(
                """
                create table content_links(
                id int primary key auto_increment,
                url varchar(100),
                type varchar(10),
                status int
                )
                """
            )

    def process_item(self, item, spider):
        # 储存到mysql 数据库
        sql = "insert into content_links values (%s,%s,%s,%s)"
        try:
            # 批量插入数据
            self.cursor.executemany(sql, item["data"])
            self.conn.commit()
            self
        except Exception as err:
            self.conn.rollback()
            print(err)
    
    # scrapy 关闭时调用以下函数
    def close_spider(self,spider):
        self.cursor.close()
        self.conn.close()

总爬取了117078个内容页链接,共花5分钟…

step3.分布式爬取每个内容页

咳咳~~重点来了,这里重新创建一个新爬虫项目,以免搞乱之前写的代码~!

刚写好程序准备测试,结果:

…

• 所以再另外写一个登录程序:
• 只要爬取之前登录一下,就可以大方的爬取数据了

from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

def login(login_url, user_url):
    chrome = webdriver.Chrome()
    # 打开登陆页面
    chrome.get(login_url)
    try:
        # 设置等待时间
        w = wait(chrome, 60)
        # 等待登录(此处可手动登录 或 编写自动登录代码)
        # 判断是否已登录
        w.until(EC.presence_of_element_located((By.CLASS_NAME, "bn-more")), message="login is failed!")
        # 打开用户面板
        chrome.get(user_url)
        w.until(EC.presence_of_element_located((By.ID, "usr-profile-nav-doulists")), message="access user page is failed!")
        # 获取cookies(列表字典类型 -> [{....},{.....},......] )
        json_cookies = chrome.get_cookies()
        cookies = {}
        for cookie in json_cookies:
            # 提取cookies中name和value的键值对组成新cookies字典
            cookies[cookie["name"]] = cookie["value"]
            # 保存到文件
        with open("chrome_cookie.txt", "w") as f:
            # 必须转换成字符串类型
            f.write(str(cookies))
    except Exception as error:
        print(error)
        return False
    chrome.close()
    return True


if __name__ == '__main__':
    login_url = "https://accounts.douban.com/passport/login"
    user_url = "https://www.douban.com/people/215290729/"
    result = login(login_url, user_url)

settings.py 中添加以下内容
编写爬虫文件之前先配置scrapy,好让scrapy可以实现分布式爬虫

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# 设置URL去重
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 设置调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 设置暂停恢复后是否继续
SCHEDULER_PERSIST = True
#SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
#SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
#SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderStack"
# 开启redis管道
ITEM_PIPELINES = {'scrapy_redis.pipelines.RedisPipeline': 400,}
# 设置显示日志等级
LOG_LEVEL = 'DEBUG'
# 设置redis服务器IP
REDIS_HOST = "192.168.2.208"
# 设置redis端口
REDIS_PORT = 6379
# 设置redis数据库编号
REDIS_DB = 1
# 连接数据库配置
REDIS_PARAMS = {
    'socket_timeout': 30,
    'socket_connect_timeout': 30,
    'retry_on_timeout': True,
    'encoding': 'utf-8',
    'db': REDIS_DB
}

• 爬虫文件

 # -*- coding: utf-8 -*-
import scrapy, ast, re, pymysql, redis, sys

sys.path.append("..")
import settings


class DoubandushuSpider(scrapy.Spider):
    name = 'doubandushu'
    allowed_domains = ['douban.com']
    # 连接redis
    redis_cli = redis.Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT)
    # 连接数据库
    conn = pymysql.connect("192.168.2.208", "root", "123456", "douban")
    # 游标
    cursor = conn.cursor()

    def start_requests(self):
        # 读取cookies
        cookies = ast.literal_eval(open("chrome_cookie.txt").read())
        # 每次读取200个内容页链接
        sql = "select id,url,type from content_links where status = 0 limit 200"
        self.cursor.execute(sql)
        urls = self.cursor.fetchall()
        # 如果urls为空表示数据库再也没有可爬取的url,跳出循环
        while len(urls) != 0:
            for id, url, type in urls:
                yield scrapy.Request(url, callback=self.parse, cookies=cookies, meta={"id": id, "type": type})
            # 每次爬取完200个内容页url都会重新获取200个
            self.cursor.execute(sql)
            urls = self.cursor.fetchall()

    def parse(self, response):
        # 获取数据
        try:
            name = response.xpath("//h1/span/text()").extract_first()
            info = response.xpath("string(//div[@id='info'])").extract_first()
            info = re.sub(r"[\n\s]", "", info)
            author = re.findall(r"作者:\s*(.+)出版社:", info)
            author = author[0] if author else None
            date = re.findall(r"出版年:\s*(\w{4})", info, re.A)
            date = date[0] if date else None
            price = re.findall(r"定价:\s*(.*[0-9])[\u4e00-\u9fa5]+:I*", info, re.A)
            try:
                price = price[0] if price else re.findall(r"定价:\s*(.*[0-9])[\u4e00-\u9fa5]*ISB*", info, re.A)[0]
            except Exception as error:
                price = 0
            score = response.xpath("//strong/text()").extract_first()
            score = score.replace(" ", "")
            if score == "":
                score = None
            rating_count = response.xpath("//a[@class='rating_people']/span/text()").extract_first()
            comment_count = response.xpath("//header//span[@class='pl']/a/text()").extract_first()
            comment_count = re.sub(r"[全部条\s]", "", comment_count)
            datas = {
                "id": response.meta["id"],
                "name": name,
                "author": author,
                "date": date,
                "price": price,
                "score": score,
                "rating_count": rating_count,
                "comment_count": comment_count,
                "type": response.meta["type"],
                "url": response.url
            }
            # 获取完数据保存数据之前,把内容页url的status设置为1
            # 保证以后获取的URL不会重复,就算重新运行爬虫也可以继续爬取status = 0的url
            self.cursor.execute("update content_links set status = 1 where id = %s", (datas['id']))
            self.conn.commit()
            yield datas
        except Exception as error:
            self.conn.rollback()
            print(error)

愉快的爬取中
飘红不是错误提示,而是日志提醒

step4:linux运行scrapy爬虫

linux中创建scrapy项目

替换settings.py

上传爬虫文件到spiders里

运行命令

然后就可以愉快的爬爬爬了

最后:怎么获取数据到本地?

so easy~~~

import redis,ast
redis_cli = redis.Redis(db=1)

while True:
    data = redis_cli.blpop("doubandushu:items")
    print(ast.literal_eval(data[1].decode()))

最后附上 豆瓣读书数据分析实战