python数据解析之bs4实战演练

需求：爬取三国演义小说所有的章节标题和章节内容；

url = 'https://www.shicimingju.com/book/sanguoyanyi.html'

思路：
- 使用通用爬虫爬取将该页面的源文件
- 解析该源文件，得到章节标题和每个章节对应的详情url地址
- 对该详情url地址进行访问得到章节的内容

from bs4 import BeautifulSoup
import requests
import lxml
import os
headers  = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.63'
}
url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
# 对首页内容进行爬取
page_text= requests.get(url=url,headers=headers).content

# 解析
# 1.实例化一个BeautifulSoup对象
soup = BeautifulSoup(page_text,'lxml')
# 2.解析 用 select 写一个层级选择器
li_list = soup.select('.book-mulu >ul >li')
fp = open('./sanguo.txt','w',encoding='utf-8')
for li in li_list:
    title = li.a.string
    detail_url = 'https://www.shicimingju.com' + li.a['href']
    # 对详情页发起请求，解析出章节内容
    detail_page_text = requests.get(url=detail_url,headers=headers).content
    #   解析出详情页中相关的章节内容
    detail_soup = BeautifulSoup(detail_page_text,'lxml')
    div_tag = detail_soup.find('div',class_='chapter_content')
    # 解析到了章节的内容
    content = div_tag.text
    fp.write(title+':'+content+'\n')
    print(title,'爬取成功！！！')