python爬取喜马拉雅有声小说

以绝世高手为例，爬取前三页的所有音频
采集地址：https://www.ximalaya.com/youshengshu/16411402/

思路只有两个：获取一级页面、获取二级页面

先一步二步看一下代码：

    def get_mes(self):
        id_list = []
        # 获取前三页的地址，range顾头不顾尾
        for i in range(1, 4):
            page_url = f'https://www.ximalaya.com/youshengshu/16411402/p{i}/'
            # 请求页面
            response = requests.get(url=page_url, headers=self.headers).content.decode('utf-8')
            # 使用正则表达式获取小说名字和id
            # 获取id是获取音频给关键
            pat = '<a title="(.*?)" href="(.*?)">'
            result = re.findall(pat, response, re.S)
            # print(result)
            for title, href in result[6:]:
                self.name_list.append(title)
                id_list.append(href.split('/')[-1])
                # self.url_list = 'https://www.ximalaya.com' + href
        return id_list

如果你觉得两个for看着不舒服也可以拆成两个方法
翻页就是通过观察它的地址：

    def get_audio(self, id_list):
        j = 0
        for i in id_list:
            audio_url = f'https://www.ximalaya.com/revision/play/v1/audio?id={i}&ptype=1'
            res = requests.get(audio_url, headers=self.headers).content.decode('utf-8')
            pat = '"src":"(.*?)"'
            result = re.findall(pat, res, re.S)
            # print(result[0])
            response = requests.get(result[0])
            if response:
                with open(f'{self.name_list[j]}.m4a', 'wb') as f:
                    f.write(response.content)
                print(f'{self.name_list[j]}爬取成功')
                j += 1

完整代码：

import requests
import re


class Spider(object):
    def __init__(self):
        # 放名字
        self.name_list = []
        # self.url_list = []
        # 请求头
        self.headers = {
            'user-agent': '这里是你的user-agent的内容'
        }

    def get_mes(self):
        id_list = []
        for i in range(1, 4):
            page_url = f'https://www.ximalaya.com/youshengshu/16411402/p{i}/'
            response = requests.get(url=page_url, headers=self.headers).content.decode('utf-8')
            pat = '<a title="(.*?)" href="(.*?)">'
            result = re.findall(pat, response, re.S)
            # print(result)
            for title, href in result[6:]:
                self.name_list.append(title)
                id_list.append(href.split('/')[-1])
                # self.url_list = 'https://www.ximalaya.com' + href
        return id_list

    def get_audio(self, id_list):
        j = 0
        for i in id_list:
            audio_url = f'https://www.ximalaya.com/revision/play/v1/audio?id={i}&ptype=1'
            res = requests.get(audio_url, headers=self.headers).content.decode('utf-8')
            pat = '"src":"(.*?)"'
            result = re.findall(pat, res, re.S)
            # print(result[0])
            response = requests.get(result[0])
            if response:
                with open(f'{self.name_list[j]}.m4a', 'wb') as f:
                    f.write(response.content)
                print(f'{self.name_list[j]}爬取成功')
                j += 1

    def run(self):
        id_list = self.get_mes()
        self.get_audio(id_list)


spider = Spider()
spider.run()

代码有不够好的地方，后面可能会以其他的例子作为改进