4 数据解析模块
解析每页的详单信息列表和解析书箱或商品详情信息
#数据解析模块
class DataParser:
# 数据解析模块-解析每页详单url列表
@staticmethod
def Parser_data_urllist(html):
#代码未完...
# 数据解析模块-解析书箱或商品详情信息
@staticmethod
def Parser_data_detail(html,number):
#代码未完...
4.1 解析每页列表url信息
分析每页的列表信息,获取列表的url,为分析详情作准备。
通过跟踪页面元素,分析a标签,最终我们采用find_all的属性查找attrs方法使用name属性itemlist-picture可以找到所有的列表对象信息。
# 数据解析模块-解析每页详单url列表
@staticmethod
def Parser_data_urllist(html):
detail_list = [] # 详情页列表
# 开始选取所有的产品列表信息URL
soup = BeautifulSoup(html, 'html.parser')
url_list = soup.find_all('a', attrs={'name': 'itemlist-picture'})
for index_url in range(len(url_list)):
#解析出的href为:'//product.dangdang.com/29505991.html'因此需加上https:
detail_url = 'https:'+url_list[index_url]['href']
detail_list.append(detail_url)
return detail_list
4.2 解析书籍或商品详情信息
@staticmethod
def Parser_data_detail(html,number):
list_data = [] # 定义当前信息列表如序号、名称...
try:
list_data.append(number + 1) # 序号
# 解析详情网页
soup_inf = BeautifulSoup(html, 'html.parser')
# 爬取书名 #product_info > div.name_info > h1
book_name = soup_inf.select('#product_info > div.name_info > h1')
book_name = book_name[0]['title'].strip()
list_data.append(book_name)
print(book_name)
# 爬取简介信息 #product_info > div.name_info > h2 > span.head_title_name
head_title_name = soup_inf.select('#product_info > div.name_info > h2 > span.head_title_name')
head_title_name = head_title_name[0].get_text().strip()
list_data.append(head_title_name)
# 爬取author信息
list_authors=[]
authors = soup_inf.select('#author > a')
for author in authors:
list_authors.append(author.get_text().strip())
list_data.append('/'.join(list_authors))
# 爬取出版社 <a dd_name="出版社">清华大学出版社</a>
pub = soup_inf.find_all('a', attrs={'dd_name': '出版社'})
pub=pub[0].get_text().strip()
list_data.append(pub)
# 出版时间 product_info > div.messbox_info > span:nth-child(3)
pubtime = soup_inf.select('div.messbox_info > span:nth-child(3)')
pubtime = pubtime[0].get_text()[5:14] #出版时间:2024年01月
list_data.append(pubtime)
# 评论排名 #product_info > div.messbox_info > div > span:nth-child(1) > span
pinlunpaiming = soup_inf.select('#product_info > div.messbox_info > div > span:nth-child(1) > span')
if len(pinlunpaiming)>0:
pinlunpaiming = pinlunpaiming[0].get_text()
list_data.append(pinlunpaiming)
else:
list_data.append(None)
# 评论数 #comm_num_down
pinlun = soup_inf.select('#comm_num_down')
pinlun = pinlun[0].get_text().strip()
list_data.append(pinlun)
#价格 #dd-price
price = soup_inf.select('#dd-price')
price = price[0].get_text()[2:].strip()
list_data.append(price)
# 爬取分类信息 #detail-category-path > span > a
category = []
book_category= soup_inf.select('#detail-category-path > span > a')
for type in book_category:
category.append(type.get_text())
string_type = '>'.join(category)
list_data.append(string_type)
#最后一个保存到类别中
list_data.append(category[len(category)-1])
except Exception as e:
print(f"An error occurred: {e}",html)
return list_data
4.2.1 爬取书名
# 爬取书名 #product_info > div.name_info > h1
book_name = soup_inf.select('#product_info > div.name_info > h1')
book_name = book_name[0]['title'].strip()
list_data.append(book_name)
print(book_name)
4.2.2 爬取简介信息
# 爬取简介信息 #product_info > div.name_info > h2 > span.head_title_name
head_title_name = soup_inf.select('#product_info > div.name_info > h2 > span.head_title_name')
head_title_name = head_title_name[0].get_text().strip()
list_data.append(head_title_name)
4.2.3 爬取author信息
# 爬取author信息
list_authors=[]
authors = soup_inf.select('#author > a')
for author in authors:
list_authors.append(author.get_text().strip())
list_data.append('/'.join(list_authors))
4.2.4 爬取出版社
# 爬取出版社 <a dd_name="出版社">清华大学出版社</a>
pub = soup_inf.find_all('a', attrs={'dd_name': '出版社'})
pub=pub[0].get_text().strip()
list_data.append(pub)
4.2.5 其他
# 出版时间 product_info > div.messbox_info > span:nth-child(3)
pubtime = soup_inf.select('div.messbox_info > span:nth-child(3)')
pubtime = pubtime[0].get_text()[5:14] #出版时间:2024年01月
list_data.append(pubtime)
# 评论排名 #product_info > div.messbox_info > div > span:nth-child(1) > span
pinlunpaiming = soup_inf.select('#product_info > div.messbox_info > div > span:nth-child(1) > span')
if len(pinlunpaiming)>0:
pinlunpaiming = pinlunpaiming[0].get_text()
list_data.append(pinlunpaiming)
else:
list_data.append(None)
# 评论数 #comm_num_down
pinlun = soup_inf.select('#comm_num_down')
pinlun = pinlun[0].get_text().strip()
list_data.append(pinlun)
#价格 #dd-price
price = soup_inf.select('#dd-price')
price = price[0].get_text()[2:].strip()
list_data.append(price)
# 爬取分类信息 #detail-category-path > span > a
category = []
book_category= soup_inf.select('#detail-category-path > span > a')
for type in book_category:
category.append(type.get_text())
string_type = '>'.join(category)
list_data.append(string_type)
#最后一个保存到类别中
list_data.append(category[len(category)-1])