python 网络爬虫及可视化之实现（三）数据解析模块

4 数据解析模块

解析每页的详单信息列表和解析书箱或商品详情信息

#数据解析模块
class DataParser:
    # 数据解析模块-解析每页详单url列表
    @staticmethod
    def Parser_data_urllist(html):
       #代码未完...
    # 数据解析模块-解析书箱或商品详情信息
    @staticmethod
    def Parser_data_detail(html,number):
       #代码未完...

4.1 解析每页列表url信息

分析每页的列表信息，获取列表的url，为分析详情作准备。

通过跟踪页面元素，分析a标签，最终我们采用find_all的属性查找attrs方法使用name属性itemlist-picture可以找到所有的列表对象信息。

 # 数据解析模块-解析每页详单url列表
    @staticmethod
    def Parser_data_urllist(html):
        detail_list = []  # 详情页列表
        # 开始选取所有的产品列表信息URL
        soup = BeautifulSoup(html, 'html.parser')
        url_list = soup.find_all('a', attrs={'name': 'itemlist-picture'})
        for index_url in range(len(url_list)):
            #解析出的href为:'//product.dangdang.com/29505991.html'因此需加上https:
            detail_url = 'https:'+url_list[index_url]['href']
            detail_list.append(detail_url)
        return detail_list

4.2 解析书籍或商品详情信息

 		@staticmethod
    def Parser_data_detail(html,number):
        list_data = []  # 定义当前信息列表如序号、名称...
        try:
            list_data.append(number + 1)  # 序号
            # 解析详情网页
            soup_inf = BeautifulSoup(html, 'html.parser')
            # 爬取书名 #product_info > div.name_info > h1
            book_name = soup_inf.select('#product_info > div.name_info > h1')
            book_name = book_name[0]['title'].strip()
            list_data.append(book_name)
            print(book_name)

            # 爬取简介信息 #product_info > div.name_info > h2 > span.head_title_name
            head_title_name = soup_inf.select('#product_info > div.name_info > h2 > span.head_title_name')
            head_title_name = head_title_name[0].get_text().strip()
            list_data.append(head_title_name)

            # 爬取author信息
            list_authors=[]
            authors = soup_inf.select('#author > a')
            for author in authors:
                list_authors.append(author.get_text().strip())
            list_data.append('/'.join(list_authors))
            
            # 爬取出版社  <a  dd_name="出版社">清华大学出版社</a>
            pub = soup_inf.find_all('a', attrs={'dd_name': '出版社'})
            pub=pub[0].get_text().strip()
            list_data.append(pub)

            # 出版时间 product_info > div.messbox_info > span:nth-child(3)
            pubtime = soup_inf.select('div.messbox_info > span:nth-child(3)')
            pubtime = pubtime[0].get_text()[5:14]  #出版时间:2024年01月
            list_data.append(pubtime)


            # 评论排名  #product_info > div.messbox_info > div > span:nth-child(1) > span
            pinlunpaiming = soup_inf.select('#product_info > div.messbox_info > div > span:nth-child(1) > span')
            if len(pinlunpaiming)>0:
                pinlunpaiming = pinlunpaiming[0].get_text()
                list_data.append(pinlunpaiming)
            else:
                list_data.append(None)

            # 评论数  #comm_num_down
            pinlun = soup_inf.select('#comm_num_down')
            pinlun = pinlun[0].get_text().strip()
            list_data.append(pinlun)

            #价格 #dd-price
            price = soup_inf.select('#dd-price')
            price = price[0].get_text()[2:].strip()
            list_data.append(price)

            # 爬取分类信息  #detail-category-path > span > a
            category = []
            book_category= soup_inf.select('#detail-category-path > span > a')
            for type in book_category:
                category.append(type.get_text())
            string_type = '>'.join(category)
            list_data.append(string_type)
            #最后一个保存到类别中
            list_data.append(category[len(category)-1])

        except Exception as e:
            print(f"An error occurred: {e}",html)
        return list_data

4.2.1 爬取书名

 # 爬取书名 #product_info > div.name_info > h1
            book_name = soup_inf.select('#product_info > div.name_info > h1')
            book_name = book_name[0]['title'].strip()
            list_data.append(book_name)
            print(book_name)

4.2.2 爬取简介信息

# 爬取简介信息 #product_info > div.name_info > h2 > span.head_title_name
            head_title_name = soup_inf.select('#product_info > div.name_info > h2 > span.head_title_name')
            head_title_name = head_title_name[0].get_text().strip()
            list_data.append(head_title_name)

4.2.3 爬取author信息

# 爬取author信息
            list_authors=[]
            authors = soup_inf.select('#author > a')
            for author in authors:
                list_authors.append(author.get_text().strip())
            list_data.append('/'.join(list_authors))

4.2.4 爬取出版社

# 爬取出版社  <a  dd_name="出版社">清华大学出版社</a>
            pub = soup_inf.find_all('a', attrs={'dd_name': '出版社'})
            pub=pub[0].get_text().strip()
            list_data.append(pub)

4.2.5 其他

            # 出版时间 product_info > div.messbox_info > span:nth-child(3)
            pubtime = soup_inf.select('div.messbox_info > span:nth-child(3)')
            pubtime = pubtime[0].get_text()[5:14]  #出版时间:2024年01月
            list_data.append(pubtime)


            # 评论排名  #product_info > div.messbox_info > div > span:nth-child(1) > span
            pinlunpaiming = soup_inf.select('#product_info > div.messbox_info > div > span:nth-child(1) > span')
            if len(pinlunpaiming)>0:
                pinlunpaiming = pinlunpaiming[0].get_text()
                list_data.append(pinlunpaiming)
            else:
                list_data.append(None)

            # 评论数  #comm_num_down
            pinlun = soup_inf.select('#comm_num_down')
            pinlun = pinlun[0].get_text().strip()
            list_data.append(pinlun)

            #价格 #dd-price
            price = soup_inf.select('#dd-price')
            price = price[0].get_text()[2:].strip()
            list_data.append(price)

            # 爬取分类信息  #detail-category-path > span > a
            category = []
            book_category= soup_inf.select('#detail-category-path > span > a')
            for type in book_category:
                category.append(type.get_text())
            string_type = '>'.join(category)
            list_data.append(string_type)
            #最后一个保存到类别中
            list_data.append(category[len(category)-1])