python网络爬虫（第五章：实战5：bs4之数据解析）

实战1. 爬取三国演义小说所有章节标题和章节内容

# https://www.shicimingju.com/book/sanguoyanyi.html
import requests
from bs4 import BeautifulSoup

if __name__ == "__main__":
    #1.指定url
    url ='https://www.shicimingju.com/book/sanguoyanyi.html'
    header = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
    }
    #2.发送get请求
    response = requests.get(url=url,headers=header).text.encode('ISO-8859-1')   #requests.get() 获取的是响应对象    #.text获取的是响应数据

    # 3.实例化BeautifulSoup对象，需要将页面源码数据加载到该对象中
    soup = BeautifulSoup(response,'lxml')
    # 4.解析章节标题
    li_list = soup.select('.book-mulu > ul > li > a')

    fp = open('./sanguoyan.txt', 'w', encoding='utf-8')  #创建txt格式的文件夹

    for li in li_list:
        #解析章节标题
        title = li.string     #li标签下的a标签中的直系内容，则使用string
        #准备解析章节内容：
        detail_url = 'https://www.shicimingju.com' + li['href']
        #对详情页发送请求，解析出章节内容
        detail_page_text = requests.get(url=detail_url,headers=header).text.encode('ISO-8859-1')
        #解析出详情页中相关的章节内容
        detail_soup = BeautifulSoup(detail_page_text,'lxml')
        #解析到了章节的内容
        detail_tag = detail_soup.find('div',class_='chapter_content').text
        
        fp.write(title+":"+ detail_tag +'\n')
        print(title,'成功抓取到！！！')