python爬虫案例——爬取豆瓣图书信息并保存

火车切片
已于 2024-07-03 10:03:16 修改
阅读量8.2k
点赞数 20
文章标签： python 爬虫
于 2021-08-01 16:05:26 首次发布
本文链接：https://blog.csdn.net/qq_45434461/article/details/119299223
版权
python爬虫案例——爬取豆瓣图书信息并保存

所需基础
- requests库的使用
- BeautifulSoup库的使用
- re库的使用和简单的正则表达式
- tqdm（进度条）库的使用
- pandas库创建DataFrame和保存Csv操作
直接上代码，注释写的比较详细
from bs4 import BeautifulSoup
import requests
import re
#import threading
#import want2url
import pandas as pd
from tqdm import tqdm

url = "https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4?start=0&type=T"\

class douban_crawler():
    send_headers = {
        "Host": "book.douban.com",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Connection": "close"
    }

    def __init__(self, url, pages):
        """
        :param url: 爬虫的最初界面，决定了要爬的书籍的类别信息
        :param pages: 要爬取的页数，豆瓣每页20本书的信息，决定了要爬取的数据量
        """
        self.url = url
        self.pages = [20*i for i in range(pages)]
        self.book_class = ""
        self.book_names = []
        self.book_nations = []
        self.book_writers = []
        self.book_scores = []
        self.book_comments = []
        self.book_sites = []
        self.book_pages = []

    def generate_urls(self):
        idx_urls = []
        #正则表达式
        page_key = re.compile(r"(\S*\?)")
        #利用正则表达式匹配出url的必须部分，后面和控制页数的变量进行拼接成索要检索的所有url列表
        #注意利用正则匹配到的返回结果为一个列表，一般需要取出列表中的值进行下面的操作
        page_main = page_key.findall(self.url)[0]
        #“合成”所有url列表,因为豆瓣的规则是每20本书放在一页中，并且用url中的start关键字控制显示的页数
        for i in self.pages:
            g_url = page_main+"start="+str(i)+"&type=T"
            idx_urls.append(g_url)
        return idx_urls

    def open_url(self, url=None):
        #如果不给需要打开的url则自动打开最初始界面（对象初始化给的界面）
        if url == None:
            #对网站发起get请求
            resp = requests.get(self.url,headers=self.send_headers)
            #获取返回信息的文本部分
            resp_text = resp.text
            #利用BS库对文本部分进行html解析，并返回解析后的界面
            soup = BeautifulSoup(resp_text, "html.parser")
            return soup
        else:
            resp = requests.get(url, headers=self.send_headers)
            resp_text = resp.text
            soup = BeautifulSoup(resp_text, "html.parser")
            return soup

    def get_main_info(self, url):
        """
        获取url列表页面能获取主要信息，不打开各个书的独立页面，
        主要信息包括：书的所属类别，作者国家，书名，每本书的索引url，书的作者，书的评分，书的简介，书的页数
        :return: 各个主要信息的存储列表

        """
        #分别为，书类别，国家，作者和简介的正则表示式
        book_class_key = re.compile(": (\D*)")
        book_nation_key = re.compile("\[(\D*?)\]")
        book_writer_key1 = re.compile("^(\D*?)/")
        book_writer_key2 = re.compile("](\D*)$")
        book_comment_key = re.compile(r"<p>(\S*)</p>")
        #创建存储主要信息的列表：因为书名是固定的，一个大页面是一个类别，所以只需要返回一次，不需要列表存储
        book_names = []
        book_pages = []
        book_nations = []
        book_writers = []
        book_comments = []
        book_scores = []
        #对url列表进行遍历并操作
        #urls = self.generate_urls()
        #为了防止耦合，最好一个函数只操作一个页面，在主函数进行对这个函数的遍历操作
        resp = requests.get(url, headers=self.send_headers) #和上面一样的操作，向url发送get请求

        resp_text = resp.text #获取返回的文本信息

        soup = BeautifulSoup(resp_text, "html.parser") #利用BS库对html格式的文本信息进行解析
        # 获取图书类别
        book_class = soup.find("h1").get_text(strip=True)
        book_class = book_class_key.findall(book_class)
        # 获取书名
        for a in soup.find_all("a"):
            try:
                # 获取书名
                res = a.get("title")
                # 获取对应的内层网站
                res_url = a.get("href")
                # 获取每本书对应的独立页面url
                if res != None:
                    book_names.append(res)
                    book_pages.append(res_url)
            except:
                pass

        """
        获取书的作者和作者国籍，因为非中国籍的形式为[国家]作者，而中国籍作者在作者名前没有[]
        所以我们用两个正则表达式分别检索，但是少数作者即使不为中国籍，也没有加[]，此类我把这类数据当作脏数据
        为了尽可能的修正这种数据带来的影响，设置判定条件为，没有[]且作者名小于五个字，为中国作者
        """
        for nation in soup.find_all("div", attrs={"class": "pub"}):
            nn = nation.get_text().strip()
            # print(nn)
            book_writer = book_writer_key1.findall(nn)[0]

            if ']' in book_writer:
                book_writers.append(book_writer_key2.findall(book_writer)[0].strip())
            else:
                book_writers.append(book_writer)

            try:
                bn = book_nation_key.findall(nn)
                if bn == [] and len(book_writer) < 5:  #中国籍作者的判定条件
                    book_nations.append("中")
                elif bn != []:
                    # print(bn)
                    book_nations.append(bn[0])
                else:
                    book_nations.append("日")
            except:
                book_nations.append("中")

        #获取书籍简介
        for comment in soup.find_all("div", attrs={"class": "info"}):
            if comment.find_all("p") == []:
                book_comments.append("无简介")
            else:
                book_comments.append(comment.find_all("p")[0].get_text())

        #获取书籍评分
        for score in soup.find_all("span", attrs={"class": "rating_nums"}):
            book_scores.append(score.get_text())

        return book_names, book_pages, book_class*20, book_writers, book_nations, book_comments, book_scores

    def get_page_numbers(self, urls):
        """
        从每个图书的独立页面中获取数据，目前只获取了页数数据
        :param urls: 从get_main_info中生成的图书独立页面url列表
        :return: 对应图书的页数数据
        """
        book_pagesnumber = []
        print("****开始获取页数信息****")
        for url in tqdm(urls):
            rrr = requests.get(url, headers=self.send_headers)
            rtext = rrr.text
            in_soup = BeautifulSoup(rtext, 'html.parser')
            # print(in_soup.text)
            page_num = re.compile(r"页数: (\d*)").findall(in_soup.text)
            #有可能有的书缺失页数信息，遇上此类情况全部将页数设置为0
            if page_num == []:
                book_pagesnumber.append(0)
            else:
                book_pagesnumber.extend(page_num)

        return book_pagesnumber

    def begin_crawl(self):
        """
        类的“主函数”只需要执行这个函数就可以完成爬虫功能
        :return: 所有的信息列表
        """
        sum_book_names = []
        sum_book_urls = []
        sum_book_class = []
        sum_book_writers = []
        sum_book_nations = []
        sum_book_comments = []
        sum_book_scores = []
        sum_book_pages = []
        urls = self.generate_urls() #生成要爬取的所有页面的url地址
        print("****开始爬取****")
        for url in tqdm(urls):
            book_names, book_urls, book_class, book_writers, book_nations, book_comments, book_scores = self.get_main_info(url)
            book_pages = self.get_page_numbers(book_urls)

            sum_book_names.extend(book_names)
            sum_book_urls.extend(book_urls)
            sum_book_class.extend(book_class)
            sum_book_writers.extend(book_writers)
            sum_book_nations.extend(book_nations)
            sum_book_comments.extend(book_comments)
            sum_book_scores.extend(book_scores)
            sum_book_pages.extend(book_pages)

        return sum_book_names, sum_book_urls, sum_book_class, sum_book_writers, sum_book_nations, sum_book_comments, sum_book_scores, sum_book_pages

    def write2csv(self):
        """
        将爬取结果写入csv文件中
        :return: 无返回值
        """
        name, url, book_class, writer, nation, comment, score, pages = self.begin_crawl()
        info_df = pd.DataFrame(columns=["name", "url", "class", "writer", "nation", "comment", "score", "pages"])
        info_df["name"] = name
        info_df["url"] = url
        info_df["class"] = book_class
        info_df["writer"] = writer
        info_df["nation"] = nation
        info_df["comment"] = comment
        info_df["score"] = score
        info_df["pages"] = pages

        info_df.to_csv(f"{book_class[0]}.csv", header=None, encoding="utf_8_sig")


if __name__ == '__main__':
    db_crawler = douban_crawler(url, 5)
    db_crawler.write2csv()